├── .github
├── actions
│ ├── set-working-directory
│ │ └── action.yml
│ └── setup-python-env
│ │ └── action.yml
└── workflows
│ ├── all-sdk-tests.yml
│ ├── build_docs.yml
│ ├── coverage.yml
│ ├── release.yml
│ └── static_docs.yml
├── .gitignore
├── .pre-commit-config.yaml
├── README.md
├── build
└── release.sh
├── dist
└── .gitkeep
├── docs
├── Makefile
├── _build
│ ├── doctrees
│ │ ├── environment.pickle
│ │ ├── index.doctree
│ │ └── source
│ │ │ ├── examples.doctree
│ │ │ ├── intro.doctree
│ │ │ ├── modules.doctree
│ │ │ ├── notdiamond.doctree
│ │ │ ├── notdiamond.llms.doctree
│ │ │ ├── notdiamond.metrics.doctree
│ │ │ ├── notdiamond.prompts.doctree
│ │ │ ├── notdiamond.toolkit.doctree
│ │ │ ├── notdiamond.toolkit.litellm.doctree
│ │ │ └── notdiamond.toolkit.rag.doctree
│ └── html
│ │ ├── .buildinfo
│ │ ├── .doctrees
│ │ ├── environment.pickle
│ │ ├── index.doctree
│ │ └── source
│ │ │ ├── intro.doctree
│ │ │ ├── notdiamond.doctree
│ │ │ ├── notdiamond.llms.doctree
│ │ │ ├── notdiamond.metrics.doctree
│ │ │ ├── notdiamond.toolkit.doctree
│ │ │ └── notdiamond.toolkit.rag.doctree
│ │ ├── .nojekyll
│ │ ├── _modules
│ │ ├── index.html
│ │ ├── notdiamond
│ │ │ ├── _init.html
│ │ │ ├── callbacks.html
│ │ │ ├── exceptions.html
│ │ │ ├── llms
│ │ │ │ ├── client.html
│ │ │ │ ├── config.html
│ │ │ │ ├── llm.html
│ │ │ │ ├── provider.html
│ │ │ │ ├── providers.html
│ │ │ │ └── request.html
│ │ │ ├── metrics
│ │ │ │ ├── metric.html
│ │ │ │ └── request.html
│ │ │ ├── prompts.html
│ │ │ ├── prompts
│ │ │ │ ├── hash.html
│ │ │ │ └── prompt.html
│ │ │ ├── toolkit
│ │ │ │ ├── custom_router.html
│ │ │ │ ├── langchain.html
│ │ │ │ ├── openai.html
│ │ │ │ └── rag
│ │ │ │ │ ├── evaluation.html
│ │ │ │ │ ├── evaluation_dataset.html
│ │ │ │ │ ├── llms.html
│ │ │ │ │ ├── testset.html
│ │ │ │ │ └── workflow.html
│ │ │ └── types.html
│ │ ├── pydantic
│ │ │ ├── _internal
│ │ │ │ └── _repr.html
│ │ │ └── main.html
│ │ ├── ragas
│ │ │ └── dataset_schema.html
│ │ └── typing.html
│ │ ├── _static
│ │ ├── _sphinx_javascript_frameworks_compat.js
│ │ ├── basic.css
│ │ ├── css
│ │ │ ├── badge_only.css
│ │ │ ├── fonts
│ │ │ │ ├── Roboto-Slab-Bold.woff
│ │ │ │ ├── Roboto-Slab-Bold.woff2
│ │ │ │ ├── Roboto-Slab-Regular.woff
│ │ │ │ ├── Roboto-Slab-Regular.woff2
│ │ │ │ ├── fontawesome-webfont.eot
│ │ │ │ ├── fontawesome-webfont.svg
│ │ │ │ ├── fontawesome-webfont.ttf
│ │ │ │ ├── fontawesome-webfont.woff
│ │ │ │ ├── fontawesome-webfont.woff2
│ │ │ │ ├── lato-bold-italic.woff
│ │ │ │ ├── lato-bold-italic.woff2
│ │ │ │ ├── lato-bold.woff
│ │ │ │ ├── lato-bold.woff2
│ │ │ │ ├── lato-normal-italic.woff
│ │ │ │ ├── lato-normal-italic.woff2
│ │ │ │ ├── lato-normal.woff
│ │ │ │ └── lato-normal.woff2
│ │ │ ├── theme.css
│ │ │ └── vendor
│ │ │ │ └── bootstrap.min.css
│ │ ├── doctools.js
│ │ ├── documentation_options.js
│ │ ├── file.png
│ │ ├── jquery.js
│ │ ├── js
│ │ │ ├── badge_only.js
│ │ │ ├── html5shiv-printshiv.min.js
│ │ │ ├── html5shiv.min.js
│ │ │ ├── searchtools.js
│ │ │ ├── theme.js
│ │ │ └── vendor
│ │ │ │ └── bootstrap.min.js
│ │ ├── language_data.js
│ │ ├── minus.png
│ │ ├── plus.png
│ │ ├── pygments.css
│ │ ├── searchtools.js
│ │ └── sphinx_highlight.js
│ │ ├── index.html
│ │ ├── objects.inv
│ │ ├── py-modindex.html
│ │ ├── search.html
│ │ ├── searchindex.js
│ │ └── source
│ │ ├── examples.html
│ │ ├── intro.html
│ │ ├── modules.html
│ │ ├── notdiamond.html
│ │ ├── notdiamond.llms.html
│ │ ├── notdiamond.metrics.html
│ │ ├── notdiamond.prompts.html
│ │ ├── notdiamond.toolkit.html
│ │ ├── notdiamond.toolkit.litellm.html
│ │ └── notdiamond.toolkit.rag.html
├── conf.py
├── index.rst
├── make.bat
└── source
│ ├── intro.rst
│ ├── notdiamond.llms.rst
│ ├── notdiamond.metrics.rst
│ ├── notdiamond.rst
│ ├── notdiamond.toolkit.rag.rst
│ └── notdiamond.toolkit.rst
├── notdiamond
├── __init__.py
├── _init.py
├── _utils.py
├── callbacks.py
├── exceptions.py
├── llms
│ ├── __init__.py
│ ├── client.py
│ ├── config.py
│ ├── providers.py
│ └── request.py
├── metrics
│ ├── __init__.py
│ ├── metric.py
│ └── request.py
├── prompts.py
├── settings.py
├── toolkit
│ ├── __init__.py
│ ├── _retry.py
│ ├── custom_router.py
│ ├── langchain.py
│ ├── openai.py
│ └── rag
│ │ ├── __init__.py
│ │ ├── document_loaders.py
│ │ ├── evaluation.py
│ │ ├── evaluation_dataset.py
│ │ ├── llms.py
│ │ ├── metrics.py
│ │ ├── testset.py
│ │ └── workflow.py
└── types.py
├── poetry.lock
├── pyproject.toml
└── tests
├── cassettes
└── test_init
│ ├── test_async_init_multi_provider_multi_model.yaml
│ ├── test_async_init_multi_provider_multi_model_azure_error.yaml
│ ├── test_async_init_multi_provider_multi_model_multi_config.yaml
│ ├── test_init_multi_provider_multi_model.yaml
│ ├── test_init_multi_provider_multi_model_azure_error.yaml
│ └── test_init_multi_provider_multi_model_multi_config.yaml
├── conftest.py
├── helpers.py
├── static
├── airbnb_tos.md
└── bbh_implicatures.json
├── test_components
├── test_llms
│ ├── cassettes
│ │ ├── test_callbacks
│ │ │ ├── test_latency_tracking_callback.yaml
│ │ │ ├── test_llm_start_callback.yaml
│ │ │ ├── test_model_select_callback[_NDInvokerClient].yaml
│ │ │ └── test_model_select_callback[_NDRouterClient].yaml
│ │ ├── test_llm
│ │ │ ├── Test_NDLLM.test_async_model_select_with_strings[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_async_model_select_with_strings[_NDClientTarget.ROUTER-_NDRouterClient].yaml
│ │ │ ├── Test_NDLLM.test_model_select_with_strings[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_model_select_with_strings[_NDClientTarget.ROUTER-_NDRouterClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_ainvoke_response_model[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_astream_response_model[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_async_tool_calling[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_invoke_response_model[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_invoke_with_curly_braces[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_openai_interface[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_stream_response_model[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_tool_calling[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_tool_calling_astream[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_ndllm_tool_calling_stream[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_preference_id[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_NDLLM.test_preference_id[_NDClientTarget.ROUTER-_NDRouterClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_create_with_default[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_create_with_latency_tracking[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_create_with_max_model_depth[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_create_with_provider_system_prompts[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_llm_with_tradeoff[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_llm_without_ndllm_configs[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_ainvoke[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_astream[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_invoke[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_invoke_claude[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_model_select[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_model_select[_NDClientTarget.ROUTER-_NDRouterClient].yaml
│ │ │ └── Test_OpenAI_style_input.test_openai_style_input_stream[_NDClientTarget.INVOKER-_NDInvokerClient].yaml
│ │ └── test_llm_request
│ │ │ ├── test_async_llm_invoke_with_latency_tracking_success.yaml
│ │ │ ├── test_custom_model_attributes.yaml
│ │ │ ├── test_llm_invoke_with_latency_tracking_nd_chat_prompt_success.yaml
│ │ │ ├── test_llm_invoke_with_latency_tracking_success.yaml
│ │ │ └── test_session_linking.yaml
│ ├── test_callbacks.py
│ ├── test_embedding_config.py
│ ├── test_llm.py
│ ├── test_llm_request.py
│ └── test_provider.py
└── test_utils.py
├── test_documentation
├── cassettes
│ ├── test_fallback_and_custom
│ │ └── test_custom_logic.yaml
│ ├── test_function_calling
│ │ ├── test_function_calling.yaml
│ │ └── test_function_calling_via_rest_api.yaml
│ ├── test_getting_started
│ │ ├── test_main_example.yaml
│ │ ├── test_model_select.yaml
│ │ ├── test_pass_array_of_messages.yaml
│ │ └── test_programatic_define_llm_configs.yaml
│ ├── test_langchain
│ │ └── test_streaming.yaml
│ ├── test_openrouter
│ │ └── test_openrouter_integration.yaml
│ ├── test_personalization
│ │ └── test_personalization.yaml
│ └── test_structured_output
│ │ ├── test_structured_output.yaml
│ │ └── test_structured_output_streaming.yaml
├── test_fallback_and_custom.py
├── test_function_calling.py
├── test_getting_started.py
├── test_langchain.py
├── test_openrouter.py
├── test_personalization.py
└── test_structured_output.py
├── test_init.py
├── test_llm_calls
├── cassettes
│ ├── test_anthropic
│ │ ├── Test_Anthropic_LLMs.test_claude_21_response_model.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_21_with_async_streaming.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_21_with_streaming.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_5_haiku_with_openai_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_5_haiku_with_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_5_sonnet_20241022_with_openai_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_5_sonnet_20241022_with_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_5_sonnet_with_openai_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_5_sonnet_with_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_7_sonnet_20250219_with_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_7_sonnet_with_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_haiku_with_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_opus_response_model.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_opus_with_openai_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_opus_with_tool_calling.yaml
│ │ ├── Test_Anthropic_LLMs.test_claude_3_sonnet_with_openai_tool_calling.yaml
│ │ └── Test_Anthropic_LLMs.test_claude_3_sonnet_with_tool_calling.yaml
│ ├── test_cohere
│ │ ├── Test_Cohere.test_cohere_command_r_plus_response_model.yaml
│ │ ├── Test_Cohere.test_cohere_command_r_plus_with_tool_calling.yaml
│ │ ├── Test_Cohere.test_cohere_command_r_response_model.yaml
│ │ └── Test_Cohere.test_cohere_command_r_with_tool_calling.yaml
│ ├── test_mistral
│ │ ├── Test_Mistral.test_codestral_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_7b_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_8x22b_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_8x7b_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_large_2402_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_large_2402_with_openai_tool_calling.yaml
│ │ ├── Test_Mistral.test_mistral_large_2402_with_tool_calling.yaml
│ │ ├── Test_Mistral.test_mistral_large_2407_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_large_2407_with_openai_tool_calling.yaml
│ │ ├── Test_Mistral.test_mistral_large_2407_with_tool_calling.yaml
│ │ ├── Test_Mistral.test_mistral_large_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_large_with_openai_tool_calling.yaml
│ │ ├── Test_Mistral.test_mistral_large_with_tool_calling.yaml
│ │ ├── Test_Mistral.test_mistral_medium_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_nemo_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_nemo_with_openai_tool_calling.yaml
│ │ ├── Test_Mistral.test_mistral_nemo_with_tool_calling.yaml
│ │ ├── Test_Mistral.test_mistral_small_response_model.yaml
│ │ ├── Test_Mistral.test_mistral_small_with_openai_tool_calling.yaml
│ │ └── Test_Mistral.test_mistral_small_with_tool_calling.yaml
│ ├── test_openai
│ │ ├── Test_OpenAI.test_async_streaming[provider0].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider10].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider11].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider12].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider13].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider14].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider15].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider16].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider17].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider18].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider19].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider1].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider20].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider2].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider3].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider4].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider5].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider6].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider7].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider8].yaml
│ │ ├── Test_OpenAI.test_async_streaming[provider9].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider0].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider10].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider11].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider12].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider13].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider14].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider15].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider16].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider1].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider2].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider3].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider4].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider5].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider6].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider7].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider8].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider9].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider0].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider10].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider11].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider12].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider13].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider14].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider15].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider16].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider1].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider2].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider3].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider4].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider5].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider6].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider7].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider8].yaml
│ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider9].yaml
│ │ ├── Test_OpenAI.test_response_model[provider0].yaml
│ │ ├── Test_OpenAI.test_response_model[provider10].yaml
│ │ ├── Test_OpenAI.test_response_model[provider11].yaml
│ │ ├── Test_OpenAI.test_response_model[provider12].yaml
│ │ ├── Test_OpenAI.test_response_model[provider13].yaml
│ │ ├── Test_OpenAI.test_response_model[provider14].yaml
│ │ ├── Test_OpenAI.test_response_model[provider15].yaml
│ │ ├── Test_OpenAI.test_response_model[provider16].yaml
│ │ ├── Test_OpenAI.test_response_model[provider17].yaml
│ │ ├── Test_OpenAI.test_response_model[provider18].yaml
│ │ ├── Test_OpenAI.test_response_model[provider19].yaml
│ │ ├── Test_OpenAI.test_response_model[provider1].yaml
│ │ ├── Test_OpenAI.test_response_model[provider20].yaml
│ │ ├── Test_OpenAI.test_response_model[provider2].yaml
│ │ ├── Test_OpenAI.test_response_model[provider3].yaml
│ │ ├── Test_OpenAI.test_response_model[provider4].yaml
│ │ ├── Test_OpenAI.test_response_model[provider5].yaml
│ │ ├── Test_OpenAI.test_response_model[provider6].yaml
│ │ ├── Test_OpenAI.test_response_model[provider7].yaml
│ │ ├── Test_OpenAI.test_response_model[provider8].yaml
│ │ ├── Test_OpenAI.test_response_model[provider9].yaml
│ │ ├── Test_OpenAI.test_streaming[provider0].yaml
│ │ ├── Test_OpenAI.test_streaming[provider10].yaml
│ │ ├── Test_OpenAI.test_streaming[provider11].yaml
│ │ ├── Test_OpenAI.test_streaming[provider12].yaml
│ │ ├── Test_OpenAI.test_streaming[provider13].yaml
│ │ ├── Test_OpenAI.test_streaming[provider14].yaml
│ │ ├── Test_OpenAI.test_streaming[provider15].yaml
│ │ ├── Test_OpenAI.test_streaming[provider16].yaml
│ │ ├── Test_OpenAI.test_streaming[provider17].yaml
│ │ ├── Test_OpenAI.test_streaming[provider18].yaml
│ │ ├── Test_OpenAI.test_streaming[provider19].yaml
│ │ ├── Test_OpenAI.test_streaming[provider1].yaml
│ │ ├── Test_OpenAI.test_streaming[provider20].yaml
│ │ ├── Test_OpenAI.test_streaming[provider2].yaml
│ │ ├── Test_OpenAI.test_streaming[provider3].yaml
│ │ ├── Test_OpenAI.test_streaming[provider4].yaml
│ │ ├── Test_OpenAI.test_streaming[provider5].yaml
│ │ ├── Test_OpenAI.test_streaming[provider6].yaml
│ │ ├── Test_OpenAI.test_streaming[provider7].yaml
│ │ ├── Test_OpenAI.test_streaming[provider8].yaml
│ │ ├── Test_OpenAI.test_streaming[provider9].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider0].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider10].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider11].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider12].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider13].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider14].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider15].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider16].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider17].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider18].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider19].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider1].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider20].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider2].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider3].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider4].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider5].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider6].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider7].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider8].yaml
│ │ ├── Test_OpenAI.test_streaming_use_stop[provider9].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider0].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider10].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider11].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider12].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider13].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider14].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider15].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider16].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider17].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider18].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider19].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider1].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider2].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider3].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider4].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider5].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider6].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider7].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider8].yaml
│ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider9].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider0].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider10].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider11].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider12].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider13].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider14].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider15].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider16].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider17].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider18].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider19].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider1].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider2].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider3].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider4].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider5].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider6].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider7].yaml
│ │ ├── Test_OpenAI.test_with_tool_calling[provider8].yaml
│ │ └── Test_OpenAI.test_with_tool_calling[provider9].yaml
│ ├── test_openai_o1
│ │ ├── test_o1_with_system_prompt[provider0].yaml
│ │ ├── test_o1_with_system_prompt[provider1].yaml
│ │ ├── test_o1_with_system_prompt[provider2].yaml
│ │ └── test_o1_with_system_prompt[provider3].yaml
│ ├── test_perplexity
│ │ ├── Test_Perplexity_LLMs.test_llama_3_1_sonar_large_128k_online.yaml
│ │ └── Test_Perplexity_LLMs.test_sonar.yaml
│ └── test_togetherai
│ │ ├── Test_TogetherAI_LLMs.test_deepseek_r1.yaml
│ │ ├── Test_TogetherAI_LLMs.test_llama_3_1_405b_instruct_turbo.yaml
│ │ ├── Test_TogetherAI_LLMs.test_llama_3_1_70b_instruct_turbo.yaml
│ │ ├── Test_TogetherAI_LLMs.test_llama_3_1_8b_instruct_turbo.yaml
│ │ ├── Test_TogetherAI_LLMs.test_llama_3_70b_chat_hf.yaml
│ │ ├── Test_TogetherAI_LLMs.test_llama_3_8b_chat_hf.yaml
│ │ ├── Test_TogetherAI_LLMs.test_mistral_7b_instruct_v02.yaml
│ │ ├── Test_TogetherAI_LLMs.test_mixtral_8x7b_instruct_v01.yaml
│ │ └── Test_TogetherAI_LLMs.test_qwen2_72b.yaml
├── test_anthropic.py
├── test_cohere.py
├── test_google.py
├── test_mistral.py
├── test_openai.py
├── test_openai_o1.py
├── test_perplexity.py
├── test_replicate.py
└── test_togetherai.py
├── test_toolkit
├── cassettes
│ ├── test_async_completion_notdiamond.yaml
│ ├── test_async_completion_notdiamond_stream.yaml
│ ├── test_completion_notdiamond.yaml
│ ├── test_completion_notdiamond_stream.yaml
│ ├── test_completion_notdiamond_tool_calling.yaml
│ ├── test_litellm
│ │ ├── test_async_completion_notdiamond.yaml
│ │ ├── test_async_completion_notdiamond_stream.yaml
│ │ ├── test_completion_notdiamond_stream.yaml
│ │ └── test_completion_notdiamond_tool_calling.yaml
│ ├── test_openai_client
│ │ ├── test_async_openai_chat_completions_create.yaml
│ │ ├── test_async_openai_chat_completions_create_stream.yaml
│ │ ├── test_async_openai_create.yaml
│ │ ├── test_async_openai_create_default_models.yaml
│ │ ├── test_openai_chat_completions_create.yaml
│ │ ├── test_openai_chat_completions_create_stream.yaml
│ │ ├── test_openai_create.yaml
│ │ └── test_openai_create_default_models.yaml
│ └── test_retry
│ │ ├── test_async_multi_model_max_retries_config.yaml
│ │ ├── test_async_multi_model_model_messages_config.yaml
│ │ ├── test_async_multi_model_multi_provider_azure_error.yaml
│ │ ├── test_async_multi_model_multi_provider_load_balance.yaml
│ │ ├── test_async_multi_model_timeout_config.yaml
│ │ ├── test_multi_model_backoff_config.yaml
│ │ ├── test_multi_model_max_retries_config.yaml
│ │ ├── test_multi_model_model_messages_config.yaml
│ │ ├── test_multi_model_multi_provider_azure_error.yaml
│ │ ├── test_multi_model_multi_provider_load_balance.yaml
│ │ ├── test_multi_model_timeout_config.yaml
│ │ ├── test_retries.yaml
│ │ ├── test_retry_wrapper[anthropic].yaml
│ │ ├── test_retry_wrapper[openai].yaml
│ │ ├── test_retry_wrapper_async[async-anthropic].yaml
│ │ ├── test_retry_wrapper_async[async-openai].yaml
│ │ ├── test_retry_wrapper_async_create[async-anthropic].yaml
│ │ ├── test_retry_wrapper_async_create[async-azure-openai].yaml
│ │ ├── test_retry_wrapper_async_create[async-openai].yaml
│ │ ├── test_retry_wrapper_async_stream[async-anthropic].yaml
│ │ ├── test_retry_wrapper_async_stream[async-azure-openai].yaml
│ │ ├── test_retry_wrapper_async_stream[async-openai].yaml
│ │ ├── test_retry_wrapper_create[anthropic].yaml
│ │ ├── test_retry_wrapper_create[azure-openai].yaml
│ │ ├── test_retry_wrapper_create[openai].yaml
│ │ ├── test_retry_wrapper_stream[anthropic].yaml
│ │ ├── test_retry_wrapper_stream[azure-openai].yaml
│ │ └── test_retry_wrapper_stream[openai].yaml
├── langchain
│ ├── cassettes
│ │ └── test_integration
│ │ │ ├── test_invokable[anthropic-claude-3-5-sonnet-20240620-langchain_anthropic.ChatAnthropic].yaml
│ │ │ ├── test_invokable[anthropic-claude-3-haiku-20240307-langchain_anthropic.ChatAnthropic].yaml
│ │ │ ├── test_invokable[anthropic-claude-3-opus-20240229-langchain_anthropic.ChatAnthropic].yaml
│ │ │ ├── test_invokable[anthropic-claude-3-sonnet-20240229-langchain_anthropic.ChatAnthropic].yaml
│ │ │ ├── test_invokable[cohere-command-r-langchain_cohere.ChatCohere].yaml
│ │ │ ├── test_invokable[cohere-command-r-plus-langchain_cohere.ChatCohere].yaml
│ │ │ ├── test_invokable[google-gemini-1.5-flash-latest-langchain_google_genai.ChatGoogleGenerativeAI].yaml
│ │ │ ├── test_invokable[google-gemini-1.5-pro-latest-langchain_google_genai.ChatGoogleGenerativeAI].yaml
│ │ │ ├── test_invokable[mistral-codestral-latest-langchain_mistralai.ChatMistralAI].yaml
│ │ │ ├── test_invokable[mistral-mistral-large-2402-langchain_mistralai.ChatMistralAI].yaml
│ │ │ ├── test_invokable[mistral-mistral-large-2407-langchain_mistralai.ChatMistralAI].yaml
│ │ │ ├── test_invokable[mistral-mistral-medium-latest-langchain_mistralai.ChatMistralAI].yaml
│ │ │ ├── test_invokable[mistral-mistral-small-latest-langchain_mistralai.ChatMistralAI].yaml
│ │ │ ├── test_invokable[mistral-open-mistral-7b-langchain_mistralai.ChatMistralAI].yaml
│ │ │ ├── test_invokable[mistral-open-mixtral-8x22b-langchain_mistralai.ChatMistralAI].yaml
│ │ │ ├── test_invokable[mistral-open-mixtral-8x7b-langchain_mistralai.ChatMistralAI].yaml
│ │ │ ├── test_invokable[openai-gpt-3.5-turbo-0125-langchain_openai.ChatOpenAI].yaml
│ │ │ ├── test_invokable[openai-gpt-4-0125-preview-langchain_openai.ChatOpenAI].yaml
│ │ │ ├── test_invokable[openai-gpt-4-0613-langchain_openai.ChatOpenAI].yaml
│ │ │ ├── test_invokable[openai-gpt-4-1106-preview-langchain_openai.ChatOpenAI].yaml
│ │ │ ├── test_invokable[openai-gpt-4-turbo-2024-04-09-langchain_openai.ChatOpenAI].yaml
│ │ │ ├── test_invokable[openai-gpt-4o-2024-05-13-langchain_openai.ChatOpenAI].yaml
│ │ │ ├── test_invokable[openai-gpt-4o-2024-08-06-langchain_openai.ChatOpenAI].yaml
│ │ │ ├── test_invokable[openai-gpt-4o-langchain_openai.ChatOpenAI].yaml
│ │ │ ├── test_invokable[openai-gpt-4o-mini-2024-07-18-langchain_openai.ChatOpenAI].yaml
│ │ │ ├── test_invokable[togetherai-Llama-3-70b-chat-hf-langchain_together.ChatTogether].yaml
│ │ │ ├── test_invokable[togetherai-Llama-3-8b-chat-hf-langchain_together.ChatTogether].yaml
│ │ │ ├── test_invokable[togetherai-Meta-Llama-3.1-405B-Instruct-Turbo-langchain_together.ChatTogether].yaml
│ │ │ ├── test_invokable[togetherai-Meta-Llama-3.1-70B-Instruct-Turbo-langchain_together.ChatTogether].yaml
│ │ │ ├── test_invokable[togetherai-Meta-Llama-3.1-8B-Instruct-Turbo-langchain_together.ChatTogether].yaml
│ │ │ ├── test_invokable[togetherai-Mistral-7B-Instruct-v0.2-langchain_together.ChatTogether].yaml
│ │ │ ├── test_invokable[togetherai-Mixtral-8x22B-Instruct-v0.1-langchain_together.ChatTogether].yaml
│ │ │ ├── test_invokable[togetherai-Mixtral-8x7B-Instruct-v0.1-langchain_together.ChatTogether].yaml
│ │ │ ├── test_invokable[togetherai-Qwen2-72B-Instruct-langchain_together.ChatTogether].yaml
│ │ │ ├── test_notdiamond_routed_runnable.yaml
│ │ │ └── test_notdiamond_routed_runnable_chain.yaml
│ ├── test_integration.py
│ └── test_unit.py
├── rag
│ ├── cassettes
│ │ └── test_evaluation
│ │ │ └── test_evaluate.yaml
│ ├── conftest.py
│ ├── test_data_gen.py
│ ├── test_evaluation.py
│ ├── test_example_workflow.py
│ └── test_workflow.py
├── test_custom_router.py
├── test_openai_client.py
└── test_retry.py
└── test_types.py
/.github/actions/set-working-directory/action.yml:
--------------------------------------------------------------------------------
1 | name: 'Set the working directory'
2 | description: 'Sets the environment variable {{ env.WORKING_DIRECTORY }} for use in subsequent actions'
3 |
4 |
5 | inputs:
6 | working_directory:
7 | description: 'Directory to set as env.WORKING_DIRECTORY'
8 | required: false
9 | default: ${{ github.workspace }}
10 |
11 |
12 | runs:
13 | using: "composite"
14 | steps:
15 | - name: set-working-directory
16 | run: |
17 | WORKING_DIRECTORY=${{ inputs.working_directory }}
18 | echo "WORKING_DIRECTORY: $WORKING_DIRECTORY"
19 | echo "WORKING_DIRECTORY=$WORKING_DIRECTORY" >> "$GITHUB_ENV"
20 | shell: bash
21 |
--------------------------------------------------------------------------------
/.github/actions/setup-python-env/action.yml:
--------------------------------------------------------------------------------
1 | name: 'Setup python environment'
2 | description: 'Install python and packages into environment using Poetry'
3 |
4 | inputs:
5 | working_directory:
6 | description: 'Directory to run the action'
7 | required: false
8 | default: ${{ github.workspace }}
9 | python_version:
10 | description: 'Python version to use.'
11 | required: false
12 | default: '3.10'
13 |
14 | runs:
15 | using: "composite"
16 | steps:
17 | - name: Install poetry
18 | run: |
19 | python3 -m pip install --upgrade pip
20 | pip3 install poetry==1.7.1
21 | shell: bash
22 | working-directory: ${{ inputs.working_directory }}
23 | - uses: actions/setup-python@v4
24 | with:
25 | cache: 'poetry'
26 | python-version: ${{ inputs.python_version }}
27 | - name: setup venv
28 | run: |
29 | poetry install
30 | echo "$(poetry env info --path)/bin" >> $GITHUB_PATH
31 | shell: bash
32 | working-directory: ${{ inputs.working_directory }}
33 |
--------------------------------------------------------------------------------
/.github/workflows/all-sdk-tests.yml:
--------------------------------------------------------------------------------
1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python
2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
3 |
4 | name: notdiamond SDK tests
5 |
6 | on:
7 | pull_request:
8 | push:
9 | branches:
10 | - "main"
11 |
12 | defaults:
13 | run:
14 | working-directory: ./
15 |
16 | jobs:
17 | build:
18 | runs-on: ubuntu-latest
19 | continue-on-error: true
20 | permissions:
21 | # Gives the action the necessary permissions for publishing new
22 | # comments in pull requests.
23 | pull-requests: write
24 | # Gives the action the necessary permissions for pushing data to the
25 | # python-coverage-comment-action branch, and for editing existing
26 | # comments (to avoid publishing multiple comments in the same PR)
27 | contents: write
28 |
29 | strategy:
30 | matrix:
31 | python-version: ["3.10", "3.11", "3.12"]
32 | steps:
33 | - uses: actions/checkout@v4
34 | - name: Set up Python ${{ matrix.python-version }}
35 | uses: actions/setup-python@v3
36 | with:
37 | python-version: ${{ matrix.python-version }}
38 | - name: Install Poetry
39 | uses: snok/install-poetry@v1
40 | with:
41 | virtualenvs-create: true
42 | virtualenvs-in-project: true
43 | installer-parallel: true
44 | - name: Load cached venv
45 | id: cached-poetry-dependencies
46 | uses: actions/cache@v4
47 | with:
48 | path: .venv
49 | key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
50 | - name: use correct env
51 | run: poetry env use ${{ matrix.python-version }}
52 | - name: Install dependencies if venv is not cached
53 | if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
54 | run: poetry install -E "create rag openai" --with test && poetry run python -m nltk.downloader all
55 | - name: Test with pytest
56 | env:
57 | NOTDIAMOND_API_KEY: ${{ secrets.NOTDIAMOND_API_KEY }}
58 | OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
59 | OPENAI_API_VERSION: ${{ secrets.OPENAI_API_VERSION }}
60 | AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_API_KEY }}
61 | AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
62 | AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
63 | ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
64 | GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
65 | COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
66 | MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
67 | TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
68 | REPLICATE_API_KEY: ${{ secrets.REPLICATE_API_KEY }}
69 | PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }}
70 | OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
71 | NOTDIAMOND_API_URL: ${{ vars.NOTDIAMOND_API_URL }}
72 | run: |
73 | PYTEST_ADDOPTS="--color=yes" poetry run coverage run --branch -m pytest -v tests/
74 | poetry run coverage report -m
75 | - name: coverage badge + comment
76 | uses: py-cov-action/python-coverage-comment-action@v3
77 | with:
78 | GITHUB_TOKEN: ${{ github.token }}
79 | # If the coverage percentage is above or equal to this value, the badge will be green.
80 | MINIMUM_GREEN: 80
81 | MINIMUM_ORANGE: 50
82 | COVERAGE_PATH: ./
83 | - name: Store Pull Request comment to be posted
84 | uses: actions/upload-artifact@v4
85 | if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true'
86 | with:
87 | # If you use a different name, update COMMENT_ARTIFACT_NAME accordingly
88 | name: python-coverage-comment-action
89 | # If you use a different name, update COMMENT_FILENAME accordingly
90 | path: python-coverage-comment-action.txt
91 |
--------------------------------------------------------------------------------
/.github/workflows/build_docs.yml:
--------------------------------------------------------------------------------
1 | # Simple workflow for deploying static content to GitHub Pages
2 | name: Build docs based on Sphinx
3 |
4 | on:
5 | # Runs on pushes targeting the default branch
6 | push:
7 | branches: ["main"]
8 |
9 | # Allows you to run this workflow manually from the Actions tab
10 | workflow_dispatch:
11 |
12 | jobs:
13 | build:
14 | environment:
15 | name: build-docs
16 | runs-on: ubuntu-22.04
17 | steps:
18 | - name: Checkout
19 | uses: actions/checkout@v4
20 |
21 | - name: Set working directory
22 | uses: ./.github/actions/set-working-directory
23 | with:
24 | working_directory: "${{ github.workspace }}"
25 |
26 | - name: Setup python environment
27 | uses: ./.github/actions/setup-python-env
28 | with:
29 | working_directory: "${{ env.WORKING_DIRECTORY }}"
30 |
31 | - name: Build docs
32 | run: |
33 | cd docs
34 | make html
35 |
36 | - uses: EndBug/add-and-commit@v9.1.1
37 | with:
38 | add: "${{ env.WORKING_DIRECTORY }}/docs/_build/html"
39 | default_author: github_actor
40 | fetch: true
41 | message: "[skip ci] Update docs"
42 | pathspec_error_handling: ignore
43 |
--------------------------------------------------------------------------------
/.github/workflows/coverage.yml:
--------------------------------------------------------------------------------
1 | # Write a test coverage comment to each pull request which reports coverage
2 | # changes due to the PR.
3 | name: Post coverage comment
4 |
5 | on:
6 | workflow_run:
7 | workflows: ["notdiamond SDK tests"]
8 | types:
9 | - completed
10 |
11 | jobs:
12 | coverage-comment:
13 | name: Run tests & display coverage
14 | runs-on: ubuntu-latest
15 | if: github.event.workflow_run.conclusion == 'success'
16 | permissions:
17 | # Gives the action the necessary permissions for publishing new
18 | # comments in pull requests.
19 | pull-requests: write
20 | # Gives the action the necessary permissions for editing existing
21 | # comments (to avoid publishing multiple comments in the same PR)
22 | contents: write
23 | # Gives the action the necessary permissions for looking up the
24 | # workflow that launched this workflow, and download the related
25 | # artifact that contains the comment to be published
26 | actions: read
27 | steps:
28 | # DO NOT run actions/checkout here, for security reasons
29 | # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
30 | - name: Post comment
31 | uses: py-cov-action/python-coverage-comment-action@v3
32 | with:
33 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
34 | GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }}
35 | # Update those if you changed the default values:
36 | # COMMENT_ARTIFACT_NAME: python-coverage-comment-action
37 | # COMMENT_FILENAME: python-coverage-comment-action.txt
38 | # If the coverage percentage is above or equal to this value, the badge will be green.
39 | MINIMUM_GREEN: 80
40 | # Same with orange. Below is red.
41 | MINIMUM_ORANGE: 50
42 |
--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
1 | name: nd-python release
2 |
3 | on:
4 | push:
5 | tags:
6 | - 'v[0-9]+.[0-9]+.[0-9]+'
7 |
8 | permissions:
9 | contents: write
10 | id-token: write
11 |
12 | jobs:
13 | release-ndserver:
14 | runs-on: ubuntu-latest
15 | steps:
16 | - uses: actions/checkout@v4
17 | - name: Set up Python 3.11
18 | id: setup-python
19 | uses: actions/setup-python@v5
20 | with:
21 | python-version: "3.11"
22 | - name: Install Poetry
23 | uses: snok/install-poetry@v1
24 | with:
25 | virtualenvs-create: true
26 | virtualenvs-in-project: true
27 | installer-parallel: true
28 | - name: poetry build
29 | run: poetry build
30 | - name: Release
31 | uses: softprops/action-gh-release@v2
32 | if: startsWith(github.ref, 'refs/tags/')
33 | - name: pypi
34 | uses: pypa/gh-action-pypi-publish@release/v1
35 | with:
36 | user: __token__
37 | password: ${{ secrets.PYPI_API_TOKEN }}
38 |
--------------------------------------------------------------------------------
/.github/workflows/static_docs.yml:
--------------------------------------------------------------------------------
1 | # Simple workflow for deploying static content to GitHub Pages
2 | name: Deploy docs to GitHub Pages
3 |
4 | on:
5 | # Runs on pushes targeting the default branch
6 | release:
7 | types: [published]
8 |
9 | # Allows you to run this workflow manually from the Actions tab
10 | workflow_dispatch:
11 |
12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
13 | permissions:
14 | contents: read
15 | pages: write
16 | id-token: write
17 |
18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
20 | concurrency:
21 | group: "pages"
22 | cancel-in-progress: false
23 |
24 | jobs:
25 | deploy:
26 | environment:
27 | name: github-pages
28 | url: ${{ steps.deployment.outputs.page_url }}
29 | runs-on: ubuntu-latest
30 | steps:
31 | - name: Checkout
32 | uses: actions/checkout@v4
33 | - name: Setup Pages
34 | uses: actions/configure-pages@v5
35 | - name: Upload artifact
36 | uses: actions/upload-pages-artifact@v3
37 | with:
38 | # Upload entire repository
39 | path: './docs/_build/html'
40 | - name: Deploy to GitHub Pages
41 | id: deployment
42 | uses: actions/deploy-pages@v4
43 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | __pycache__/
2 | **/.DS_Store
3 | .env
4 | .envrc
5 | .python-version
6 | dist/
7 | !dist/.gitkeep
8 | /dist
9 | .vscode/
10 | venv/
11 | .venv/
12 | .idea/
13 |
--------------------------------------------------------------------------------
/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
1 | repos:
2 | - repo: https://github.com/pre-commit/pre-commit-hooks
3 | rev: v4.4.0
4 | hooks:
5 | - id: check-yaml
6 | - id: end-of-file-fixer
7 | - id: trailing-whitespace
8 |
9 | - repo: https://github.com/timothycrosley/isort
10 | rev: 5.13.2
11 | hooks:
12 | - id: isort
13 | args: ["--profile", "black", "--filter-files"]
14 | exclude: ^(cookbooks|docs)
15 |
16 | - repo: https://github.com/psf/black
17 | rev: 23.3.0
18 | hooks:
19 | - id: black
20 | exclude: ^(cookbooks|docs)
21 |
22 | - repo: https://github.com/PyCQA/flake8
23 | rev: 7.0.0
24 | hooks:
25 | - id: flake8
26 | args: [--max-line-length=130]
27 | exclude: ^(cookbooks|docs|tests/conftest.py)
28 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # notdiamond-python
2 |
3 | Not Diamond is an AI model router that automatically determines which LLM is best-suited to respond to any query, improving LLM output quality by combining multiple LLMs into a **meta-model** that learns when to call each LLM.
4 |
5 | ## Key features
6 |
7 | - **[Maximize output quality](https://notdiamond.readme.io/docs/quickstart)**: Not Diamond [outperforms every foundation model](https://notdiamond.readme.io/docs/benchmark-performance) on major evaluation benchmarks by always calling the best model for every prompt.
8 | - **[Reduce cost and latency](https://notdiamond.readme.io/docs/cost-and-latency-tradeoffs)**: Not Diamond lets you define intelligent cost and latency tradeoffs to efficiently leverage smaller and cheaper models without degrading quality.
9 | - **[Train your own router](https://notdiamond.readme.io/docs/router-training-quickstart)**: Not Diamond lets you train your own custom routers optimized to your data and use case.
10 | - **[Python](https://python.notdiamond.ai/), [TypeScript](https://www.npmjs.com/package/notdiamond), and [REST API](https://notdiamond.readme.io/reference/api-introduction) support**: Not Diamond works across a variety of stacks.
11 |
12 | ## Installation
13 |
14 | **Python**: Requires **Python 3.10+**. It's recommended that you create and activate a [virtualenv](https://virtualenv.pypa.io/en/latest/) prior to installing the package. For this example, we'll be installing the optional additional `create` dependencies, which you can learn more about [here](https://notdiamond.readme.io/docs/model_select-vs-create).
15 |
16 | ```shell
17 | pip install notdiamond[create]
18 | ```
19 |
20 | ### Apple Silicon
21 |
22 | Make sure to install `libmagic` if you intend to use the `create` or `rag` extras:
23 |
24 | ```bash
25 | brew install libmagic
26 | ```
27 |
28 | ## Setting up
29 |
30 | Create a `.env` file with your [Not Diamond API key](https://app.notdiamond.ai/keys) and the [API keys of the models](https://notdiamond.readme.io/docs/api-keys) you want to route between:
31 |
32 | ```shell
33 | NOTDIAMOND_API_KEY = "YOUR_NOTDIAMOND_API_KEY"
34 | OPENAI_API_KEY = "YOUR_OPENAI_API_KEY"
35 | ANTHROPIC_API_KEY = "YOUR_ANTHROPIC_API_KEY"
36 | ```
37 |
38 | ## Sending your first Not Diamond API request
39 |
40 | Create a new file in the same directory as your `.env` file and copy and run the code below (you can toggle between Python and TypeScript in the top left of the code block):
41 |
42 | ```python
43 | from notdiamond import NotDiamond
44 |
45 | # Define the Not Diamond routing client
46 | client = NotDiamond()
47 |
48 | # The best LLM is determined by Not Diamond based on the messages and specified models
49 | result, session_id, provider = client.chat.completions.create(
50 | messages=[
51 | {"role": "system", "content": "You are a helpful assistant."},
52 | {"role": "user", "content": "Concisely explain merge sort."} # Adjust as desired
53 | ],
54 | model=['openai/gpt-3.5-turbo', 'openai/gpt-4o', 'anthropic/claude-3-5-sonnet-20240620']
55 | )
56 |
57 | print("ND session ID: ", session_id) # A unique ID of Not Diamond's recommendation
58 | print("LLM called: ", provider.model) # The LLM routed to
59 | print("LLM output: ", result.content) # The LLM response
60 | ```
61 |
--------------------------------------------------------------------------------
/build/release.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | EXPECTED_ARGS=1
4 | if [ $# -ne $EXPECTED_ARGS ]; then
5 | echo "Invalid number of arguments. Expected: $EXPECTED_ARGS"
6 | exit 1
7 | fi
8 |
9 | VALID_VERSIONS=("major" "minor" "patch")
10 | if [[ ! " ${VALID_VERSIONS[@]} " =~ " $1 " ]]; then
11 | echo "Invalid version type. Expected one of: ${VALID_VERSIONS[*]}"
12 | exit 1
13 | fi
14 |
15 | git checkout main && git pull
16 | poetry version $1
17 | TAG_NAME=v$(poetry version -s)
18 | git add pyproject.toml
19 | git commit -m "Bump version to $TAG_NAME"
20 | git push origin main
21 |
22 | git tag $TAG_NAME && git push origin tag $TAG_NAME
23 |
--------------------------------------------------------------------------------
/dist/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/dist/.gitkeep
--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
1 | # Minimal makefile for Sphinx documentation
2 | #
3 |
4 | # You can set these variables from the command line, and also
5 | # from the environment for the first two.
6 | SPHINXOPTS ?=
7 | SPHINXBUILD ?= sphinx-build
8 | SOURCEDIR = .
9 | BUILDDIR = _build
10 |
11 | # Put it first so that "make" without argument is like "make help".
12 | help:
13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
14 |
15 | .PHONY: help Makefile
16 |
17 | # Catch-all target: route all unknown targets to Sphinx using the new
18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
19 | %: Makefile
20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
21 |
--------------------------------------------------------------------------------
/docs/_build/doctrees/environment.pickle:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/environment.pickle
--------------------------------------------------------------------------------
/docs/_build/doctrees/index.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/index.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/examples.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/examples.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/intro.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/intro.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/modules.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/modules.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/notdiamond.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/notdiamond.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/notdiamond.llms.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/notdiamond.llms.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/notdiamond.metrics.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/notdiamond.metrics.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/notdiamond.prompts.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/notdiamond.prompts.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/notdiamond.toolkit.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/notdiamond.toolkit.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/notdiamond.toolkit.litellm.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/notdiamond.toolkit.litellm.doctree
--------------------------------------------------------------------------------
/docs/_build/doctrees/source/notdiamond.toolkit.rag.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/doctrees/source/notdiamond.toolkit.rag.doctree
--------------------------------------------------------------------------------
/docs/_build/html/.buildinfo:
--------------------------------------------------------------------------------
1 | # Sphinx build info version 1
2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
3 | config: 204a5c462923e8038e5918346d8d885f
4 | tags: 645f666f9bcd5a90fca523b33c5a78b7
5 |
--------------------------------------------------------------------------------
/docs/_build/html/.doctrees/environment.pickle:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/.doctrees/environment.pickle
--------------------------------------------------------------------------------
/docs/_build/html/.doctrees/index.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/.doctrees/index.doctree
--------------------------------------------------------------------------------
/docs/_build/html/.doctrees/source/intro.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/.doctrees/source/intro.doctree
--------------------------------------------------------------------------------
/docs/_build/html/.doctrees/source/notdiamond.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/.doctrees/source/notdiamond.doctree
--------------------------------------------------------------------------------
/docs/_build/html/.doctrees/source/notdiamond.llms.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/.doctrees/source/notdiamond.llms.doctree
--------------------------------------------------------------------------------
/docs/_build/html/.doctrees/source/notdiamond.metrics.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/.doctrees/source/notdiamond.metrics.doctree
--------------------------------------------------------------------------------
/docs/_build/html/.doctrees/source/notdiamond.toolkit.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/.doctrees/source/notdiamond.toolkit.doctree
--------------------------------------------------------------------------------
/docs/_build/html/.doctrees/source/notdiamond.toolkit.rag.doctree:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/.doctrees/source/notdiamond.toolkit.rag.doctree
--------------------------------------------------------------------------------
/docs/_build/html/.nojekyll:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/.nojekyll
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/badge_only.css:
--------------------------------------------------------------------------------
1 | .clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}}
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/Roboto-Slab-Bold.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/Roboto-Slab-Bold.woff
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/Roboto-Slab-Bold.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/Roboto-Slab-Bold.woff2
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/Roboto-Slab-Regular.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/Roboto-Slab-Regular.woff
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/Roboto-Slab-Regular.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/Roboto-Slab-Regular.woff2
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/fontawesome-webfont.eot:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/fontawesome-webfont.eot
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/fontawesome-webfont.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/fontawesome-webfont.ttf
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/fontawesome-webfont.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/fontawesome-webfont.woff
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/fontawesome-webfont.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/fontawesome-webfont.woff2
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/lato-bold-italic.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/lato-bold-italic.woff
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/lato-bold-italic.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/lato-bold-italic.woff2
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/lato-bold.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/lato-bold.woff
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/lato-bold.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/lato-bold.woff2
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/lato-normal-italic.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/lato-normal-italic.woff
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/lato-normal-italic.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/lato-normal-italic.woff2
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/lato-normal.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/lato-normal.woff
--------------------------------------------------------------------------------
/docs/_build/html/_static/css/fonts/lato-normal.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/css/fonts/lato-normal.woff2
--------------------------------------------------------------------------------
/docs/_build/html/_static/documentation_options.js:
--------------------------------------------------------------------------------
1 | const DOCUMENTATION_OPTIONS = {
2 | VERSION: '0.4.4',
3 | LANGUAGE: 'en',
4 | COLLAPSE_INDEX: false,
5 | BUILDER: 'html',
6 | FILE_SUFFIX: '.html',
7 | LINK_SUFFIX: '.html',
8 | HAS_SOURCE: false,
9 | SOURCELINK_SUFFIX: '.txt',
10 | NAVIGATION_WITH_KEYS: false,
11 | SHOW_SEARCH_SUMMARY: true,
12 | ENABLE_SEARCH_SHORTCUTS: true,
13 | };
--------------------------------------------------------------------------------
/docs/_build/html/_static/file.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/file.png
--------------------------------------------------------------------------------
/docs/_build/html/_static/js/badge_only.js:
--------------------------------------------------------------------------------
1 | !function(e){var t={};function r(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,r),o.l=!0,o.exports}r.m=e,r.c=t,r.d=function(e,t,n){r.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:n})},r.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.t=function(e,t){if(1&t&&(e=r(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var n=Object.create(null);if(r.r(n),Object.defineProperty(n,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var o in e)r.d(n,o,function(t){return e[t]}.bind(null,o));return n},r.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(t,"a",t),t},r.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r.p="",r(r.s=4)}({4:function(e,t,r){}});
--------------------------------------------------------------------------------
/docs/_build/html/_static/js/html5shiv.min.js:
--------------------------------------------------------------------------------
1 | /**
2 | * @preserve HTML5 Shiv 3.7.3 | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed
3 | */
4 | !function(a,b){function c(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x",d.insertBefore(c.lastChild,d.firstChild)}function d(){var a=t.elements;return"string"==typeof a?a.split(" "):a}function e(a,b){var c=t.elements;"string"!=typeof c&&(c=c.join(" ")),"string"!=typeof a&&(a=a.join(" ")),t.elements=c+" "+a,j(b)}function f(a){var b=s[a[q]];return b||(b={},r++,a[q]=r,s[r]=b),b}function g(a,c,d){if(c||(c=b),l)return c.createElement(a);d||(d=f(c));var e;return e=d.cache[a]?d.cache[a].cloneNode():p.test(a)?(d.cache[a]=d.createElem(a)).cloneNode():d.createElem(a),!e.canHaveChildren||o.test(a)||e.tagUrn?e:d.frag.appendChild(e)}function h(a,c){if(a||(a=b),l)return a.createDocumentFragment();c=c||f(a);for(var e=c.frag.cloneNode(),g=0,h=d(),i=h.length;i>g;g++)e.createElement(h[g]);return e}function i(a,b){b.cache||(b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag()),a.createElement=function(c){return t.shivMethods?g(c,a,b):b.createElem(c)},a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+d().join().replace(/[\w\-:]+/g,function(a){return b.createElem(a),b.frag.createElement(a),'c("'+a+'")'})+");return n}")(t,b.frag)}function j(a){a||(a=b);var d=f(a);return!t.shivCSS||k||d.hasCSS||(d.hasCSS=!!c(a,"article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}mark{background:#FF0;color:#000}template{display:none}")),l||i(a,d),a}var k,l,m="3.7.3-pre",n=a.html5||{},o=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,p=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,q="_html5shiv",r=0,s={};!function(){try{var a=b.createElement("a");a.innerHTML="",k="hidden"in a,l=1==a.childNodes.length||function(){b.createElement("a");var a=b.createDocumentFragment();return"undefined"==typeof a.cloneNode||"undefined"==typeof a.createDocumentFragment||"undefined"==typeof a.createElement}()}catch(c){k=!0,l=!0}}();var t={elements:n.elements||"abbr article aside audio bdi canvas data datalist details dialog figcaption figure footer header hgroup main mark meter nav output picture progress section summary template time video",version:m,shivCSS:n.shivCSS!==!1,supportsUnknownElements:l,shivMethods:n.shivMethods!==!1,type:"default",shivDocument:j,createElement:g,createDocumentFragment:h,addElements:e};a.html5=t,j(b),"object"==typeof module&&module.exports&&(module.exports=t)}("undefined"!=typeof window?window:this,document);
--------------------------------------------------------------------------------
/docs/_build/html/_static/minus.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/minus.png
--------------------------------------------------------------------------------
/docs/_build/html/_static/plus.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/_static/plus.png
--------------------------------------------------------------------------------
/docs/_build/html/objects.inv:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/docs/_build/html/objects.inv
--------------------------------------------------------------------------------
/docs/conf.py:
--------------------------------------------------------------------------------
1 | import os
2 | import sys
3 | from pathlib import Path
4 |
5 | import toml
6 |
7 | # Configuration file for the Sphinx documentation builder.
8 | #
9 | # For the full list of built-in configuration values, see the documentation:
10 | # https://www.sphinx-doc.org/en/master/usage/configuration.html
11 |
12 |
13 | _DIR = Path(__file__).parent.absolute()
14 | with (_DIR.parents[0] / "pyproject.toml").open("r") as f:
15 | data = toml.load(f)
16 |
17 | # -- Project information -----------------------------------------------------
18 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
19 |
20 | project = "NotDiamond"
21 | copyright = "2024, NotDiamond"
22 | author = "NotDiamond"
23 |
24 | version = data["tool"]["poetry"]["version"]
25 | release = version
26 |
27 | html_title = project + " " + version
28 | html_last_updated_fmt = "%b %d, %Y"
29 | html_show_sourcelink = False
30 |
31 | sys.path.insert(0, os.path.abspath(".."))
32 |
33 | # -- General configuration ---------------------------------------------------
34 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
35 |
36 | extensions = [
37 | "sphinx.ext.autodoc",
38 | "sphinx.ext.doctest",
39 | "sphinx.ext.intersphinx",
40 | "sphinx.ext.todo",
41 | "sphinx.ext.coverage",
42 | "sphinx.ext.mathjax",
43 | "sphinx.ext.ifconfig",
44 | "sphinx.ext.viewcode",
45 | "sphinx.ext.githubpages",
46 | "sphinx.ext.napoleon",
47 | ]
48 |
49 | # Napoleon settings
50 | napoleon_google_docstring = True
51 | napoleon_numpy_docstring = True
52 | napoleon_include_init_with_doc = True
53 | napoleon_include_private_with_doc = True
54 | napoleon_include_special_with_doc = True
55 | napoleon_use_admonition_for_examples = False
56 | napoleon_use_admonition_for_notes = False
57 | napoleon_use_admonition_for_references = False
58 | napoleon_use_ivar = False
59 | napoleon_use_param = True
60 | napoleon_use_rtype = True
61 | autodoc_pydantic_model_show_json = False
62 | autodoc_pydantic_field_list_validators = False
63 | autodoc_pydantic_config_members = False
64 | autodoc_pydantic_model_show_config_summary = False
65 | autodoc_pydantic_model_show_validator_members = False
66 | autodoc_pydantic_model_show_validator_summary = False
67 | autodoc_pydantic_model_signature_prefix = "class"
68 | autodoc_pydantic_field_signature_prefix = "param"
69 | autodoc_member_order = "groupwise"
70 | autoclass_content = "both"
71 | autodoc_typehints_format = "short"
72 | autodoc_typehints = "both"
73 |
74 | templates_path = ["_templates"]
75 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
76 |
77 | # List of patterns, relative to source directory, that match files and
78 | # directories to ignore when looking for source files.
79 | # This pattern also affects html_static_path and html_extra_path.
80 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
81 |
82 |
83 | # -- Options for HTML output -------------------------------------------------
84 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
85 |
86 | html_theme = "sphinx_rtd_theme"
87 | html_static_path = ["_static"]
88 | html_use_index = False
89 |
90 | # generate autosummary even if no references
91 | autosummary_generate = True
92 |
93 | html_copy_source = False
94 | html_show_sourcelink = False
95 |
96 | autodoc_mock_imports = ["notdiamond.toolkit._retry"]
97 |
--------------------------------------------------------------------------------
/docs/index.rst:
--------------------------------------------------------------------------------
1 | .. NotDiamond documentation master file, created by
2 | sphinx-quickstart on Wed Apr 24 11:09:36 2024.
3 | You can adapt this file completely to your liking, but it should at least
4 | contain the root `toctree` directive.
5 |
6 | ========================================
7 | Not Diamond Python library documentation
8 | ========================================
9 |
10 | .. toctree::
11 | :maxdepth: 2
12 | :caption: Contents:
13 |
14 | source/intro.rst
15 | source/notdiamond.rst
16 |
--------------------------------------------------------------------------------
/docs/make.bat:
--------------------------------------------------------------------------------
1 | @ECHO OFF
2 |
3 | pushd %~dp0
4 |
5 | REM Command file for Sphinx documentation
6 |
7 | if "%SPHINXBUILD%" == "" (
8 | set SPHINXBUILD=sphinx-build
9 | )
10 | set SOURCEDIR=.
11 | set BUILDDIR=_build
12 |
13 | %SPHINXBUILD% >NUL 2>NUL
14 | if errorlevel 9009 (
15 | echo.
16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
17 | echo.installed, then set the SPHINXBUILD environment variable to point
18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you
19 | echo.may add the Sphinx directory to PATH.
20 | echo.
21 | echo.If you don't have Sphinx installed, grab it from
22 | echo.https://www.sphinx-doc.org/
23 | exit /b 1
24 | )
25 |
26 | if "%1" == "" goto help
27 |
28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
29 | goto end
30 |
31 | :help
32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
33 |
34 | :end
35 | popd
36 |
--------------------------------------------------------------------------------
/docs/source/intro.rst:
--------------------------------------------------------------------------------
1 | Getting started with Not Diamond
2 | ================================
3 |
4 | Not Diamond is an AI model router that automatically determines which LLM is best-suited to respond to any query, improving LLM output quality by combining multiple LLMs into a **meta-model** that learns when to call each LLM.
5 |
6 | ============
7 | Key features
8 | ============
9 |
10 | - `Maximize output quality `_: Not Diamond `outperforms every foundation model `_ on major evaluation benchmarks by always calling the best model for every prompt.
11 | - `Reduce cost and latency `_: Not Diamond lets you define intelligent cost and latency tradeoffs to efficiently leverage smaller and cheaper models without degrading quality.
12 | - `Train your own router `_: Not Diamond lets you train your own custom routers optimized to your data and use case.
13 | - `Python `_, `TypeScript `_, and `REST API `_ support: Not Diamond works across a variety of stacks.
14 |
15 | ============
16 | Installation
17 | ============
18 |
19 | **Python**: Requires **Python 3.10+**. It’s recommended that you create and activate a `virtualenv `_ prior to installing the package. For this example, we'll be installing the optional additional `create` dependencies, which you can learn more about `here `_.
20 |
21 | .. code:: shell
22 |
23 | pip install notdiamond[create]
24 |
25 | ==========
26 | Setting up
27 | ==========
28 |
29 | Create a `.env` file with your `Not Diamond API key `_ and the `API keys of the models `_ you want to route between:
30 |
31 | .. code:: shell
32 |
33 | NOTDIAMOND_API_KEY = "YOUR_NOTDIAMOND_API_KEY"
34 | OPENAI_API_KEY = "YOUR_OPENAI_API_KEY"
35 | ANTHROPIC_API_KEY = "YOUR_ANTHROPIC_API_KEY"
36 |
37 | ==========================================
38 | Sending your first Not Diamond API request
39 | ==========================================
40 |
41 | Create a new file in the same directory as your `.env` file and copy and run the code below (you can toggle between Python and TypeScript in the top left of the code block):
42 |
43 | .. code:: python
44 |
45 | from notdiamond import NotDiamond
46 |
47 | # Define the Not Diamond routing client
48 | client = NotDiamond()
49 |
50 | # The best LLM is determined by Not Diamond based on the messages and specified models
51 | result, session_id, provider = client.chat.completions.create(
52 | messages=[
53 | {"role": "system", "content": "You are a helpful assistant."},
54 | {"role": "user", "content": "Consiely explain merge sort."} # Adjust as desired
55 | ],
56 | model=['openai/gpt-3.5-turbo', 'openai/gpt-4o', 'anthropic/claude-3-5-sonnet-20240620']
57 | )
58 |
59 | print("ND session ID: ", session_id) # A unique ID of Not Diamond's recommendation
60 | print("LLM called: ", provider.model) # The LLM routed to
61 | print("LLM output: ", result.content) # The LLM response
62 |
--------------------------------------------------------------------------------
/docs/source/notdiamond.llms.rst:
--------------------------------------------------------------------------------
1 | notdiamond.llms
2 | =======================
3 |
4 |
5 | notdiamond.llms.client
6 | --------------------------
7 |
8 | .. autoclass:: notdiamond.llms.client.NotDiamond
9 | :members:
10 | :undoc-members:
11 | :show-inheritance:
12 |
13 | notdiamond.llms.config
14 | -------------------------------
15 |
16 | .. automodule:: notdiamond.llms.config
17 | :members:
18 | :undoc-members:
19 | :show-inheritance:
20 |
21 | notdiamond.llms.providers
22 | --------------------------------
23 |
24 | .. automodule:: notdiamond.llms.providers
25 | :members:
26 | :undoc-members:
27 | :show-inheritance:
28 | :no-index:
29 |
30 | notdiamond.llms.request
31 | ------------------------------
32 |
33 | .. automodule:: notdiamond.llms.request
34 | :members:
35 | :undoc-members:
36 | :show-inheritance:
37 |
--------------------------------------------------------------------------------
/docs/source/notdiamond.metrics.rst:
--------------------------------------------------------------------------------
1 | notdiamond.metrics
2 | ==================
3 |
4 |
5 | notdiamond.metrics.metric
6 | -------------------------
7 |
8 | .. automodule:: notdiamond.metrics.metric
9 | :members:
10 | :undoc-members:
11 | :show-inheritance:
12 |
13 | notdiamond.metrics.request
14 | --------------------------
15 |
16 | .. automodule:: notdiamond.metrics.request
17 | :members:
18 | :undoc-members:
19 | :show-inheritance:
20 |
--------------------------------------------------------------------------------
/docs/source/notdiamond.rst:
--------------------------------------------------------------------------------
1 | NotDiamond package
2 | ==================
3 |
4 | Subpackages
5 | -----------
6 |
7 | .. toctree::
8 | :maxdepth: 4
9 |
10 | notdiamond.llms
11 | notdiamond.metrics
12 | notdiamond.toolkit
13 |
14 | Submodules
15 | ----------
16 |
17 | .. toctree::
18 | :maxdepth: 4
19 |
20 | notdiamond._utils
21 | notdiamond.callbacks
22 | notdiamond.exceptions
23 | notdiamond.prompts
24 | notdiamond.types
25 |
26 | Module contents
27 | --------------
28 |
29 | .. automodule:: notdiamond
30 | :members:
31 | :undoc-members:
32 | :show-inheritance:
33 |
34 | .. autofunction:: notdiamond.init
35 |
--------------------------------------------------------------------------------
/docs/source/notdiamond.toolkit.rag.rst:
--------------------------------------------------------------------------------
1 | notdiamond.toolkit.rag
2 | ======================
3 |
4 | notdiamond.toolkit.rag.document_loaders
5 | ---------------------------------------
6 |
7 | .. automodule:: notdiamond.toolkit.rag.document_loaders
8 | :members:
9 | :undoc-members:
10 | :show-inheritance:
11 |
12 | notdiamond.toolkit.rag.evaluation_dataset
13 | -----------------------------------------
14 |
15 | .. automodule:: notdiamond.toolkit.rag.evaluation_dataset
16 | :members:
17 | :undoc-members:
18 | :show-inheritance:
19 | :special-members: __init__
20 | :exclude-members: from_list, generation_prompt, generator_llm
21 |
22 | notdiamond.toolkit.rag.evaluation
23 | -----------------------------------------
24 |
25 | .. automodule:: notdiamond.toolkit.rag.evaluation
26 | :members:
27 | :undoc-members:
28 | :show-inheritance:
29 |
30 | notdiamond.toolkit.rag.llms
31 | -------------------------------
32 |
33 | .. automodule:: notdiamond.toolkit.rag.llms
34 | :members:
35 | :undoc-members:
36 | :show-inheritance:
37 |
38 | notdiamond.toolkit.rag.metrics
39 | -------------------------------
40 |
41 | .. automodule:: notdiamond.toolkit.rag.metrics
42 | :members:
43 | :undoc-members:
44 | :show-inheritance:
45 |
46 | notdiamond.toolkit.rag.testset
47 | -------------------------------
48 |
49 | .. automodule:: notdiamond.toolkit.rag.testset
50 | :members:
51 | :undoc-members:
52 | :show-inheritance:
53 |
54 | notdiamond.toolkit.rag.workflow
55 | -------------------------------
56 |
57 | .. automodule:: notdiamond.toolkit.rag.workflow
58 | :members:
59 | :undoc-members:
60 | :show-inheritance:
61 |
--------------------------------------------------------------------------------
/docs/source/notdiamond.toolkit.rst:
--------------------------------------------------------------------------------
1 | notdiamond.toolkit package
2 | ======================
3 |
4 | Submodules
5 | ----------
6 |
7 | .. toctree::
8 | :maxdepth: 4
9 |
10 | notdiamond.toolkit.rag
11 |
12 | Module contents
13 | --------------
14 |
15 | .. automodule:: notdiamond.toolkit
16 | :members:
17 | :undoc-members:
18 | :show-inheritance:
19 |
20 | notdiamond.toolkit.custom_router
21 | --------------------------------
22 |
23 | .. autoclass:: notdiamond.toolkit.custom_router.CustomRouter
24 | :members:
25 | :undoc-members:
26 | :show-inheritance:
27 |
--------------------------------------------------------------------------------
/notdiamond/__init__.py:
--------------------------------------------------------------------------------
1 | from notdiamond.llms.client import NotDiamond # noqa: F401
2 | from notdiamond.llms.config import LLMConfig # noqa: F401
3 | from notdiamond.metrics.metric import Metric # noqa: F401
4 |
5 | __all__ = []
6 | try:
7 | import importlib.util
8 |
9 | spec = importlib.util.find_spec("openai")
10 | if spec is not None:
11 | from ._init import init # noqa: F401
12 |
13 | __all__ = ["init"]
14 | else:
15 | __all__ = []
16 | except ImportError:
17 | pass
18 |
--------------------------------------------------------------------------------
/notdiamond/callbacks.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from typing import Any
3 |
4 | from notdiamond._utils import _module_check
5 | from notdiamond.llms.config import LLMConfig
6 |
7 | LOGGER = logging.getLogger(__name__)
8 | LOGGER.setLevel(logging.INFO)
9 |
10 |
11 | def _nd_callback_handler_factory():
12 | try:
13 | BaseCallbackHandler = _module_check(
14 | "langchain_core.callbacks.base", "BaseCallbackHandler"
15 | )
16 | except (ModuleNotFoundError, ImportError) as ierr:
17 | log_msg = (
18 | "Attempted to instantiate NDLLMBaseCallbackHandler but langchain is not installed. ",
19 | f"Model callbacks not available. {ierr}",
20 | )
21 | LOGGER.warning(log_msg)
22 | return
23 |
24 | class _NDLLMBaseCallbackHandler(BaseCallbackHandler):
25 | """
26 | Base callback handler for NotDiamond LLMs.
27 | Accepts all of the langchain_core callbacks and adds new ones.
28 | """
29 |
30 | def on_model_select(
31 | self, model_provider: LLMConfig, model_name: str
32 | ) -> Any:
33 | """
34 | Called when a model is selected.
35 | """
36 |
37 | def on_latency_tracking(
38 | self,
39 | session_id: str,
40 | model_provider: LLMConfig,
41 | tokens_per_second: float,
42 | ):
43 | """
44 | Called when latency tracking is enabled.
45 | """
46 |
47 | def on_api_error(self, error_message: str):
48 | """
49 | Called when an NotDiamond API error occurs.
50 | """
51 |
52 | return _NDLLMBaseCallbackHandler
53 |
54 |
55 | NDLLMBaseCallbackHandler = _nd_callback_handler_factory()
56 |
--------------------------------------------------------------------------------
/notdiamond/exceptions.py:
--------------------------------------------------------------------------------
1 | class UnsupportedLLMProvider(Exception):
2 | """The exception class for unsupported LLM provider"""
3 |
4 |
5 | class UnsupportedEmbeddingProvider(Exception):
6 | """The exception class for unsupported Embedding provider"""
7 |
8 |
9 | class InvalidApiKey(Exception):
10 | """The exception class for InvalidApiKey"""
11 |
12 |
13 | class MissingApiKey(Exception):
14 | """The exception class for MissingApiKey"""
15 |
16 |
17 | class MissingLLMConfigs(Exception):
18 | """The exception class for empty LLM provider configs array"""
19 |
20 |
21 | class ApiError(Exception):
22 | """The exception class for any ApiError"""
23 |
24 |
25 | class CreateUnavailableError(Exception):
26 | """The exception class raised when `notdiamond[create]` is not available"""
27 |
--------------------------------------------------------------------------------
/notdiamond/llms/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/notdiamond/llms/__init__.py
--------------------------------------------------------------------------------
/notdiamond/metrics/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/notdiamond/metrics/__init__.py
--------------------------------------------------------------------------------
/notdiamond/metrics/metric.py:
--------------------------------------------------------------------------------
1 | from typing import Optional
2 |
3 | from notdiamond import settings
4 | from notdiamond.exceptions import ApiError
5 | from notdiamond.llms.config import LLMConfig
6 | from notdiamond.metrics.request import feedback_request
7 | from notdiamond.types import NDApiKeyValidator
8 |
9 |
10 | class Metric:
11 | def __init__(self, metric: Optional[str] = "accuracy"):
12 | self.metric = metric
13 |
14 | def __call__(self):
15 | return self.metric
16 |
17 | def feedback(
18 | self,
19 | session_id: str,
20 | llm_config: LLMConfig,
21 | value: int,
22 | notdiamond_api_key: Optional[str] = None,
23 | _user_agent: str = None,
24 | ):
25 | if notdiamond_api_key is None:
26 | notdiamond_api_key = settings.NOTDIAMOND_API_KEY
27 | NDApiKeyValidator(api_key=notdiamond_api_key)
28 | if value not in [0, 1]:
29 | raise ApiError("Invalid feedback value. It must be 0 or 1.")
30 |
31 | return feedback_request(
32 | session_id=session_id,
33 | llm_config=llm_config,
34 | feedback_payload=self.request_payload(value),
35 | notdiamond_api_key=notdiamond_api_key,
36 | _user_agent=_user_agent,
37 | )
38 |
39 | def request_payload(self, value: int):
40 | return {self.metric: value}
41 |
--------------------------------------------------------------------------------
/notdiamond/metrics/request.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from typing import Dict
3 |
4 | import requests
5 |
6 | from notdiamond import settings
7 | from notdiamond._utils import _default_headers
8 | from notdiamond.exceptions import ApiError
9 | from notdiamond.llms.config import LLMConfig
10 | from notdiamond.types import FeedbackRequestPayload
11 |
12 | LOGGER = logging.getLogger(__name__)
13 | LOGGER.setLevel(logging.INFO)
14 |
15 |
16 | def feedback_request(
17 | session_id: str,
18 | llm_config: LLMConfig,
19 | feedback_payload: Dict[str, int],
20 | notdiamond_api_key: str,
21 | nd_api_url: str = settings.NOTDIAMOND_API_URL,
22 | _user_agent: str = settings.DEFAULT_USER_AGENT,
23 | ) -> bool:
24 | url = f"{nd_api_url}/v2/report/metrics/feedback"
25 |
26 | payload: FeedbackRequestPayload = {
27 | "session_id": session_id,
28 | "provider": llm_config.prepare_for_request(),
29 | "feedback": feedback_payload,
30 | }
31 |
32 | headers = _default_headers(notdiamond_api_key, _user_agent)
33 |
34 | try:
35 | response = requests.post(url, json=payload, headers=headers)
36 | except Exception as e:
37 | raise ApiError(f"ND API error for feedback: {e}")
38 |
39 | if response.status_code != 200:
40 | LOGGER.error(
41 | f"ND API feedback error: failed to report feedback with status {response.status_code}. {response.text}"
42 | )
43 | return False
44 |
45 | return True
46 |
--------------------------------------------------------------------------------
/notdiamond/prompts.py:
--------------------------------------------------------------------------------
1 | import logging
2 | import re
3 | from typing import Dict, List
4 |
5 | from notdiamond.llms.config import LLMConfig
6 | from notdiamond.llms.providers import is_o1_model
7 |
8 | LOGGER = logging.getLogger(__name__)
9 | LOGGER.setLevel(logging.INFO)
10 |
11 |
12 | def inject_system_prompt(
13 | messages: List[Dict[str, str]], system_prompt: str
14 | ) -> List[Dict[str, str]]:
15 | """
16 | Add a system prompt to an OpenAI-style message list. If a system prompt is already present, replace it.
17 | """
18 | new_messages = []
19 | found = False
20 | for msg in messages:
21 | # t7: replace the first system prompt with the new one
22 | if msg["role"] == "system" and not found:
23 | new_messages.append({"role": "system", "content": system_prompt})
24 | found = True
25 | else:
26 | new_messages.append(msg)
27 | if not found:
28 | new_messages.insert(0, {"role": "system", "content": system_prompt})
29 | return new_messages
30 |
31 |
32 | def _curly_escape(text: str) -> str:
33 | """
34 | Escape curly braces in the text, but only for single occurrences of alphabetic characters.
35 | This function will not escape double curly braces or non-alphabetic characters.
36 | """
37 | return re.sub(r"(? List[Dict[str, str]]:
43 | if is_o1_model(llm):
44 | translated_messages = []
45 | for msg in messages:
46 | if msg["role"] == "system":
47 | translated_messages.append(
48 | {"role": "user", "content": msg["content"]}
49 | )
50 | else:
51 | translated_messages.append(msg)
52 | return translated_messages
53 | return messages
54 |
--------------------------------------------------------------------------------
/notdiamond/toolkit/__init__.py:
--------------------------------------------------------------------------------
1 | from .custom_router import CustomRouter # noqa
2 |
--------------------------------------------------------------------------------
/notdiamond/toolkit/rag/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/c18c622c900cc40ee574553777b686e5c81cbcac/notdiamond/toolkit/rag/__init__.py
--------------------------------------------------------------------------------
/notdiamond/toolkit/rag/document_loaders.py:
--------------------------------------------------------------------------------
1 | from langchain_community.document_loaders import DirectoryLoader # noqa
2 |
--------------------------------------------------------------------------------
/notdiamond/toolkit/rag/evaluation_dataset.py:
--------------------------------------------------------------------------------
1 | from dataclasses import dataclass
2 | from typing import Dict, List, Union, overload
3 |
4 | from ragas import MultiTurnSample, SingleTurnSample
5 | from ragas.dataset_schema import RagasDataset
6 |
7 |
8 | class RAGSample(SingleTurnSample):
9 | """
10 | Represents RAG evaluation samples.
11 |
12 | Attributes:
13 | user_input (Optional[str]): The input query from the user.
14 | retrieved_contexts (Optional[List[str]]): List of contexts retrieved for the query.
15 | reference_contexts (Optional[List[str]]): List of reference contexts for the query.
16 | response (Optional[str]): The generated response for the query.
17 | generation_prompt (str): The input prompt to the generator LLM.
18 | generator_llm (str): The LLM used to generate the response.
19 | multi_responses (Optional[List[str]]): List of multiple responses generated for the query.
20 | reference (Optional[str]): The reference answer for the query.
21 | rubric (Optional[Dict[str, str]]): Evaluation rubric for the sample.
22 | """
23 |
24 | generation_prompt: str
25 | generator_llm: str
26 |
27 |
28 | @dataclass
29 | class RAGEvaluationDataset(RagasDataset[RAGSample]):
30 | """
31 | Represents a dataset of RAG evaluation samples.
32 |
33 | Attributes:
34 | samples (List[BaseSample]): A list of evaluation samples.
35 |
36 | Methods:
37 | validate_samples(samples): Validates that all samples are of the same type.
38 | get_sample_type(): Returns the type of the samples in the dataset.
39 | to_hf_dataset(): Converts the dataset to a Hugging Face Dataset.
40 | to_pandas(): Converts the dataset to a pandas DataFrame.
41 | features(): Returns the features of the samples.
42 | from_list(mapping): Creates an EvaluationDataset from a list of dictionaries.
43 | from_dict(mapping): Creates an EvaluationDataset from a dictionary.
44 | to_csv(path): Converts the dataset to a CSV file.
45 | to_jsonl(path): Converts the dataset to a JSONL file.
46 | from_jsonl(path): Creates an EvaluationDataset from a JSONL file.
47 | """
48 |
49 | @overload
50 | def __getitem__(self, idx: int) -> RAGSample:
51 | ...
52 |
53 | @overload
54 | def __getitem__(self, idx: slice) -> "RAGEvaluationDataset":
55 | ...
56 |
57 | def __getitem__(
58 | self, idx: Union[int, slice]
59 | ) -> Union[RAGSample, "RAGEvaluationDataset"]:
60 | if isinstance(idx, int):
61 | return self.samples[idx]
62 | elif isinstance(idx, slice):
63 | return type(self)(samples=self.samples[idx])
64 | else:
65 | raise TypeError("Index must be int or slice")
66 |
67 | def is_multi_turn(self) -> bool:
68 | return False
69 |
70 | def to_list(self) -> List[Dict]:
71 | rows = [sample.to_dict() for sample in self.samples]
72 | return rows
73 |
74 | @classmethod
75 | def from_list(cls, data: List[Dict]):
76 | samples = []
77 | if all(
78 | "user_input" in item and isinstance(data[0]["user_input"], list)
79 | for item in data
80 | ):
81 | samples.extend(MultiTurnSample(**sample) for sample in data)
82 | else:
83 | samples.extend(SingleTurnSample(**sample) for sample in data)
84 | return cls(samples=samples)
85 |
86 | def __repr__(self) -> str:
87 | return f"RAGEvaluationDataset(features={self.features()}, len={len(self.samples)})"
88 |
--------------------------------------------------------------------------------
/notdiamond/toolkit/rag/llms.py:
--------------------------------------------------------------------------------
1 | from typing import Union
2 |
3 | from langchain_cohere import CohereEmbeddings
4 | from langchain_mistralai import MistralAIEmbeddings
5 | from langchain_openai import OpenAIEmbeddings
6 | from ragas.embeddings import HuggingfaceEmbeddings, LangchainEmbeddingsWrapper
7 | from ragas.llms import LangchainLLMWrapper
8 |
9 | from ...exceptions import UnsupportedEmbeddingProvider
10 | from ...llms.client import NotDiamond
11 | from ...llms.config import EmbeddingConfig, LLMConfig
12 |
13 |
14 | def get_llm(llm_config_or_str: Union[LLMConfig, str]) -> LangchainLLMWrapper:
15 | """
16 | Build the LLM object compatible with evaluation metrics.
17 |
18 | Parameters:
19 | llm_config_or_str (Union[LLMConfig, str]): a LLMConfig object or a model string
20 | that specifies the LLM to construct.
21 | """
22 | if isinstance(llm_config_or_str, str):
23 | llm_config = LLMConfig.from_string(llm_config_or_str)
24 | else:
25 | llm_config = llm_config_or_str
26 |
27 | lc_llm = NotDiamond._llm_from_config(llm_config)
28 | return LangchainLLMWrapper(lc_llm)
29 |
30 |
31 | def get_embedding(
32 | embedding_model_config_or_str: Union[EmbeddingConfig, str]
33 | ) -> Union[LangchainEmbeddingsWrapper, HuggingfaceEmbeddings]:
34 | """
35 | Build the embedding model object compatible with evaluation metrics.
36 |
37 | Parameters:
38 | embedding_model_config_or_str (Union[EmbeddingConfig, str]): an EmbeddingConfig object
39 | or an embedding model string that specifies the embedding model to construct.
40 | """
41 | if isinstance(embedding_model_config_or_str, str):
42 | embedding_config = EmbeddingConfig.from_string(
43 | embedding_model_config_or_str
44 | )
45 | else:
46 | embedding_config = embedding_model_config_or_str
47 |
48 | if embedding_config.provider == "openai":
49 | lc_embedding = OpenAIEmbeddings(
50 | model=embedding_config.model, **embedding_config.kwargs
51 | )
52 |
53 | elif embedding_config.provider == "cohere":
54 | lc_embedding = CohereEmbeddings(
55 | model=embedding_config.model, **embedding_config.kwargs
56 | )
57 |
58 | elif embedding_config.provider == "mistral":
59 | lc_embedding = MistralAIEmbeddings(
60 | model=embedding_config.model, **embedding_config.kwargs
61 | )
62 |
63 | elif embedding_config.provider == "huggingface":
64 | return HuggingfaceEmbeddings(model_name=embedding_config.model)
65 |
66 | else:
67 | raise UnsupportedEmbeddingProvider(
68 | f"Embedding model {str(embedding_config)} not supported."
69 | )
70 | return LangchainEmbeddingsWrapper(lc_embedding)
71 |
--------------------------------------------------------------------------------
/notdiamond/toolkit/rag/metrics.py:
--------------------------------------------------------------------------------
1 | from ragas.metrics import ( # noqa
2 | AspectCritic,
3 | BleuScore,
4 | ContextEntityRecall,
5 | ContextPrecision,
6 | ContextRecall,
7 | ExactMatch,
8 | FactualCorrectness,
9 | Faithfulness,
10 | FaithfulnesswithHHEM,
11 | InstanceRubrics,
12 | )
13 | from ragas.metrics import ( # noqa
14 | LLMContextPrecisionWithoutReference as ContextPrecisionWithoutReference,
15 | )
16 | from ragas.metrics import ( # noqa
17 | LLMContextPrecisionWithReference,
18 | LLMContextRecall,
19 | NoiseSensitivity,
20 | NonLLMContextPrecisionWithReference,
21 | NonLLMContextRecall,
22 | NonLLMStringSimilarity,
23 | ResponseRelevancy,
24 | RougeScore,
25 | RubricsScore,
26 | SemanticSimilarity,
27 | SimpleCriteriaScore,
28 | StringPresence,
29 | )
30 |
--------------------------------------------------------------------------------
/notdiamond/types.py:
--------------------------------------------------------------------------------
1 | from typing import Any, Dict, List
2 |
3 | from pydantic import BaseModel, field_validator
4 |
5 | from notdiamond.exceptions import InvalidApiKey, MissingApiKey
6 |
7 |
8 | class NDApiKeyValidator(BaseModel):
9 | api_key: str
10 |
11 | @field_validator("api_key", mode="before")
12 | @classmethod
13 | def api_key_must_be_a_string(cls, v) -> str:
14 | if not isinstance(v, str):
15 | raise InvalidApiKey("ND API key should be a string")
16 | return v
17 |
18 | @field_validator("api_key", mode="after")
19 | @classmethod
20 | def string_must_not_be_empty(cls, v):
21 | if len(v) == 0:
22 | raise MissingApiKey("ND API key should be longer than 0")
23 | return v
24 |
25 |
26 | class ModelSelectRequestPayload(BaseModel):
27 | prompt_template: str
28 | formatted_prompt: str
29 | components: Dict[str, Dict]
30 | llm_configs: List[Dict]
31 | metric: str
32 | max_model_depth: int
33 |
34 |
35 | class FeedbackRequestPayload(BaseModel):
36 | session_id: str
37 | provider: Dict[str, Any]
38 | feedback: Dict[str, int]
39 |
--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
1 | [tool.poetry]
2 | name = "notdiamond"
3 | version = "0.4.4"
4 | description = "Not Diamond Python SDK"
5 | authors = ["Not Diamond "]
6 | readme = "README.md"
7 | homepage = "https://python.notdiamond.ai/"
8 | repository = "https://github.com/Not-Diamond/notdiamond-python"
9 | documentation = "https://notdiamond.readme.io/docs/"
10 |
11 | [tool.poetry.dependencies]
12 | python = ">3.10,<3.13"
13 | python-dotenv = "^1.0.1"
14 | aiohttp = "^3.9.3"
15 | pandas = "^2.2.2"
16 | pydantic-partial = "^0.5.5"
17 | cryptography = ">=44.0.1"
18 | h11 = ">=0.16.0"
19 | tiktoken = ">=0.8.0,<0.9.0"
20 | langchain = { version = ">=0.3.22", optional = true }
21 | langchain-google-genai = { version = ">=2.1.2", optional = true }
22 | langchain-openai = { version = ">=0.3.9", optional = true }
23 | langchain-anthropic = { version = ">=0.3.10", optional = true }
24 | langchain-community = { version = ">=0.3.20", optional = true }
25 | langchain-mistralai = { version = ">=0.2.10", optional = true }
26 | langchain-together = { version = ">=0.3.0", optional = true }
27 | langchain-cohere = { version = ">=0.4.3", optional = true }
28 | requests = ">=2.31"
29 | optuna = { version = "^4.1.0", optional = true }
30 | ragas = { version = "^0.2.6", optional = true }
31 | unstructured = { extras = [
32 | "local-inference",
33 | ], version = "^0.16.5", optional = true }
34 | onnxruntime = { version = "<1.20", optional = true }
35 | python-multipart = { version = "0.0.18", extras = ["rag"] }
36 | llama-index = { version = "^0.12.25", optional = true }
37 |
38 | [tool.poetry.group.test.dependencies]
39 | pytest = "^8.0.0"
40 | pytest-asyncio = "^0.23.6"
41 | pytest-mock = "^3.14.0"
42 | openai = ">=1.66.3"
43 | coverage = "^7.6.0"
44 | pytest-recording = "^0.13.2"
45 | pytest-timeout = "^2.3.1"
46 |
47 | [tool.poetry.group.dev.dependencies]
48 | ruff = "^0.3.1"
49 | pydantic = "^2.5.3"
50 | pre-commit = "^3.7.0"
51 | black = "^24.3.0"
52 | flake8 = "^7.0.0"
53 | poetry-pre-commit-plugin = "^0.1.2"
54 | isort = "^5.13.2"
55 | nox = "^2024.3.2"
56 | pytest-mock = "^3.14.0"
57 |
58 |
59 | [tool.poetry.group.docs.dependencies]
60 | sphinx = "^7.3.7"
61 | autodoc = "^0.5.0"
62 | sphinx-rtd-theme = "^2.0.0"
63 | toml = "^0.10.2"
64 |
65 | [tool.poetry.extras]
66 | create = [
67 | "langchain-cohere",
68 | "langchain-anthropic",
69 | "langchain-community",
70 | "langchain-mistralai",
71 | "langchain-google-genai",
72 | "langchain-openai",
73 | "langchain-together",
74 | "langchain",
75 | ]
76 | openai = ["openai"]
77 | rag = [
78 | "ragas",
79 | "unstructured",
80 | "optuna",
81 | "onnxruntime",
82 | "langchain-cohere",
83 | "langchain-anthropic",
84 | "langchain-community",
85 | "langchain-mistralai",
86 | "langchain-google-genai",
87 | "langchain-openai",
88 | "langchain-together",
89 | "langchain",
90 | "llama-index",
91 | ]
92 |
93 | [build-system]
94 | requires = ["poetry-core"]
95 | build-backend = "poetry.core.masonry.api"
96 |
97 | [tool.pytest.ini_options]
98 | markers = ["longrun"]
99 |
100 | [tool.black]
101 | line-length = 79
102 | target-version = ['py39']
103 | include = '\.pyi?$'
104 | exclude = '''
105 | /(
106 | \.git
107 | | \.mypy_cache
108 | | \.venv
109 | | _build
110 | )/
111 | '''
112 |
113 | [tool.flake8]
114 | max-line-length = 130
115 | extend-ignore = [
116 | "D203",
117 | "E203",
118 | "E251",
119 | "E266",
120 | "E302",
121 | "E305",
122 | "E401",
123 | "E402",
124 | "E501",
125 | "F401",
126 | "F403",
127 | "W503",
128 | ]
129 | exclude = [".git", "__pycache__", "dist", "cookbooks"]
130 | max-complexity = 10
131 |
132 | [tool.isort]
133 | atomic = true
134 | profile = "black"
135 | line_length = 79
136 | skip_gitignore = true
137 |
138 | [tool.coverage.run]
139 | branch = true
140 | relative_files = true
141 |
142 | [tool.coverage.report]
143 | exclude_also = [
144 | # Don't complain about abstract methods, they aren't run:
145 | "@(abc\\.)?abstractmethod",
146 | ]
147 |
--------------------------------------------------------------------------------
/tests/helpers.py:
--------------------------------------------------------------------------------
1 | def stream_chunks(ichunk):
2 | n_chunks = 0
3 | all_empty = True
4 | for chunk in ichunk:
5 | assert chunk.type == "AIMessageChunk"
6 | assert isinstance(chunk.content, str)
7 | all_empty &= len(chunk.content) == 0
8 | n_chunks += 1
9 | if n_chunks >= 5:
10 | break
11 | assert not all_empty
12 |
13 |
14 | async def astream_chunks(async_ichunk):
15 | n_chunks = 0
16 | all_empty = True
17 | async for chunk in async_ichunk:
18 | assert chunk.type == "AIMessageChunk"
19 | assert isinstance(chunk.content, str)
20 | all_empty &= len(chunk.content) == 0
21 | n_chunks += 1
22 | if n_chunks >= 5:
23 | break
24 | assert not all_empty
25 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_callbacks/test_model_select_callback[_NDInvokerClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Hello, what''s your name?"}],
4 | "llm_providers": [{"provider": "openai", "model": "gpt-4-turbo-preview", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}, {"provider": "openai", "model": "gpt-3.5-turbo", "is_custom": false,
7 | "context_length": null, "input_price": null, "output_price": null, "latency":
8 | null}], "metric": "accuracy", "max_model_depth": 2, "hash_content": false}'
9 | headers:
10 | Accept:
11 | - '*/*'
12 | Accept-Encoding:
13 | - gzip, deflate, zstd
14 | Connection:
15 | - keep-alive
16 | Content-Length:
17 | - '471'
18 | User-Agent:
19 | - Python-SDK/0.3.41
20 | content-type:
21 | - application/json
22 | method: POST
23 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
24 | response:
25 | body:
26 | string: '{"providers":[{"provider":"openai","model":"gpt-3.5-turbo"}],"session_id":"89489a7c-694f-46da-91a2-c7a763837d8e"}'
27 | headers:
28 | CF-RAY:
29 | - 925cfab0cc25c228-VIE
30 | Connection:
31 | - keep-alive
32 | Content-Type:
33 | - application/json
34 | Date:
35 | - Tue, 25 Mar 2025 08:18:44 GMT
36 | Server:
37 | - cloudflare
38 | Transfer-Encoding:
39 | - chunked
40 | alt-svc:
41 | - h3=":443"; ma=86400
42 | cf-cache-status:
43 | - DYNAMIC
44 | content-length:
45 | - '113'
46 | rndr-id:
47 | - 9709566f-eeb7-4edb
48 | vary:
49 | - Accept-Encoding
50 | x-render-origin-server:
51 | - uvicorn
52 | status:
53 | code: 200
54 | message: OK
55 | version: 1
56 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_callbacks/test_model_select_callback[_NDRouterClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Hello, what''s your name?"}],
4 | "llm_providers": [{"provider": "openai", "model": "gpt-4-turbo-preview", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}, {"provider": "openai", "model": "gpt-3.5-turbo", "is_custom": false,
7 | "context_length": null, "input_price": null, "output_price": null, "latency":
8 | null}], "metric": "accuracy", "max_model_depth": 2, "hash_content": false}'
9 | headers:
10 | Accept:
11 | - '*/*'
12 | Accept-Encoding:
13 | - gzip, deflate, zstd
14 | Connection:
15 | - keep-alive
16 | Content-Length:
17 | - '471'
18 | User-Agent:
19 | - Python-SDK/0.3.41
20 | content-type:
21 | - application/json
22 | method: POST
23 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
24 | response:
25 | body:
26 | string: '{"providers":[{"provider":"openai","model":"gpt-3.5-turbo"}],"session_id":"35b450dc-40d8-4559-b0c2-2c9adbee9356"}'
27 | headers:
28 | CF-RAY:
29 | - 925cfaac2cc1c268-VIE
30 | Connection:
31 | - keep-alive
32 | Content-Type:
33 | - application/json
34 | Date:
35 | - Tue, 25 Mar 2025 08:18:43 GMT
36 | Server:
37 | - cloudflare
38 | Transfer-Encoding:
39 | - chunked
40 | alt-svc:
41 | - h3=":443"; ma=86400
42 | cf-cache-status:
43 | - DYNAMIC
44 | content-length:
45 | - '113'
46 | rndr-id:
47 | - c59c84ee-6b17-458b
48 | vary:
49 | - Accept-Encoding
50 | x-render-origin-server:
51 | - uvicorn
52 | status:
53 | code: 200
54 | message: OK
55 | version: 1
56 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_async_model_select_with_strings[_NDClientTarget.INVOKER-_NDInvokerClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Write me a song about goldfish
4 | on the moon"}], "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo",
5 | "is_custom": false, "context_length": null, "input_price": null, "output_price":
6 | null, "latency": null}, {"provider": "openai", "model": "gpt-4", "is_custom":
7 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
8 | null}, {"provider": "anthropic", "model": "claude-2.1", "is_custom": false,
9 | "context_length": null, "input_price": null, "output_price": null, "latency":
10 | null}, {"provider": "google", "model": "gemini-pro", "is_custom": false, "context_length":
11 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
12 | "accuracy", "max_model_depth": 4, "hash_content": true}'
13 | headers:
14 | User-Agent:
15 | - Python-SDK/0.3.41
16 | content-type:
17 | - application/json
18 | method: POST
19 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
20 | response:
21 | body:
22 | string: '{"providers":[{"provider":"openai","model":"gpt-4"}],"session_id":"7a3032f4-cf00-45d3-9f45-f1f61830abbf"}'
23 | headers:
24 | CF-RAY:
25 | - 925cfae498ddc747-VIE
26 | Connection:
27 | - keep-alive
28 | Content-Type:
29 | - application/json
30 | Date:
31 | - Tue, 25 Mar 2025 08:18:52 GMT
32 | Server:
33 | - cloudflare
34 | Transfer-Encoding:
35 | - chunked
36 | Vary:
37 | - Accept-Encoding
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | content-length:
43 | - '105'
44 | rndr-id:
45 | - cd53e938-6152-4e22
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 200
50 | message: OK
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_async_model_select_with_strings[_NDClientTarget.ROUTER-_NDRouterClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Write me a song about goldfish
4 | on the moon"}], "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo",
5 | "is_custom": false, "context_length": null, "input_price": null, "output_price":
6 | null, "latency": null}, {"provider": "openai", "model": "gpt-4", "is_custom":
7 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
8 | null}, {"provider": "anthropic", "model": "claude-2.1", "is_custom": false,
9 | "context_length": null, "input_price": null, "output_price": null, "latency":
10 | null}, {"provider": "google", "model": "gemini-pro", "is_custom": false, "context_length":
11 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
12 | "accuracy", "max_model_depth": 4, "hash_content": true}'
13 | headers:
14 | User-Agent:
15 | - Python-SDK/0.3.41
16 | content-type:
17 | - application/json
18 | method: POST
19 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
20 | response:
21 | body:
22 | string: '{"providers":[{"provider":"openai","model":"gpt-4"}],"session_id":"3625a798-925a-4435-9855-1ab93d0ed74f"}'
23 | headers:
24 | CF-RAY:
25 | - 925cfadf29645b4e-VIE
26 | Connection:
27 | - keep-alive
28 | Content-Type:
29 | - application/json
30 | Date:
31 | - Tue, 25 Mar 2025 08:18:52 GMT
32 | Server:
33 | - cloudflare
34 | Transfer-Encoding:
35 | - chunked
36 | Vary:
37 | - Accept-Encoding
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | content-length:
43 | - '105'
44 | rndr-id:
45 | - 1e7b850b-bbb4-4ef2
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 200
50 | message: OK
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_model_select_with_strings[_NDClientTarget.INVOKER-_NDInvokerClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Write me a song about goldfish
4 | on the moon"}], "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo",
5 | "is_custom": false, "context_length": null, "input_price": null, "output_price":
6 | null, "latency": null}, {"provider": "openai", "model": "gpt-4", "is_custom":
7 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
8 | null}, {"provider": "anthropic", "model": "claude-2.1", "is_custom": false,
9 | "context_length": null, "input_price": null, "output_price": null, "latency":
10 | null}, {"provider": "google", "model": "gemini-pro", "is_custom": false, "context_length":
11 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
12 | "accuracy", "max_model_depth": 4, "hash_content": true}'
13 | headers:
14 | Accept:
15 | - '*/*'
16 | Accept-Encoding:
17 | - gzip, deflate, zstd
18 | Connection:
19 | - keep-alive
20 | Content-Length:
21 | - '779'
22 | User-Agent:
23 | - Python-SDK/0.3.41
24 | content-type:
25 | - application/json
26 | method: POST
27 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
28 | response:
29 | body:
30 | string: '{"providers":[{"provider":"openai","model":"gpt-4"}],"session_id":"5d496dbd-2fc8-4044-a018-fec52ad9f39c"}'
31 | headers:
32 | CF-RAY:
33 | - 925cfadabdd3c1f9-VIE
34 | Connection:
35 | - keep-alive
36 | Content-Type:
37 | - application/json
38 | Date:
39 | - Tue, 25 Mar 2025 08:18:51 GMT
40 | Server:
41 | - cloudflare
42 | Transfer-Encoding:
43 | - chunked
44 | alt-svc:
45 | - h3=":443"; ma=86400
46 | cf-cache-status:
47 | - DYNAMIC
48 | content-length:
49 | - '105'
50 | rndr-id:
51 | - 2d7f2dc4-0459-47a7
52 | vary:
53 | - Accept-Encoding
54 | x-render-origin-server:
55 | - uvicorn
56 | status:
57 | code: 200
58 | message: OK
59 | version: 1
60 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_model_select_with_strings[_NDClientTarget.ROUTER-_NDRouterClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Write me a song about goldfish
4 | on the moon"}], "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo",
5 | "is_custom": false, "context_length": null, "input_price": null, "output_price":
6 | null, "latency": null}, {"provider": "openai", "model": "gpt-4", "is_custom":
7 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
8 | null}, {"provider": "anthropic", "model": "claude-2.1", "is_custom": false,
9 | "context_length": null, "input_price": null, "output_price": null, "latency":
10 | null}, {"provider": "google", "model": "gemini-pro", "is_custom": false, "context_length":
11 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
12 | "accuracy", "max_model_depth": 4, "hash_content": true}'
13 | headers:
14 | Accept:
15 | - '*/*'
16 | Accept-Encoding:
17 | - gzip, deflate, zstd
18 | Connection:
19 | - keep-alive
20 | Content-Length:
21 | - '779'
22 | User-Agent:
23 | - Python-SDK/0.3.41
24 | content-type:
25 | - application/json
26 | method: POST
27 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
28 | response:
29 | body:
30 | string: '{"providers":[{"provider":"openai","model":"gpt-4"}],"session_id":"43eac436-56f1-4ec9-9dcd-4d51d1b31baf"}'
31 | headers:
32 | CF-RAY:
33 | - 925cfad5eebad100-SOF
34 | Connection:
35 | - keep-alive
36 | Content-Type:
37 | - application/json
38 | Date:
39 | - Tue, 25 Mar 2025 08:18:50 GMT
40 | Server:
41 | - cloudflare
42 | Transfer-Encoding:
43 | - chunked
44 | alt-svc:
45 | - h3=":443"; ma=86400
46 | cf-cache-status:
47 | - DYNAMIC
48 | content-length:
49 | - '105'
50 | rndr-id:
51 | - 5072108e-7aad-4c90
52 | vary:
53 | - Accept-Encoding
54 | x-render-origin-server:
55 | - uvicorn
56 | status:
57 | code: 200
58 | message: OK
59 | version: 1
60 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_preference_id[_NDClientTarget.INVOKER-_NDInvokerClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"name": "test"}'
4 | headers:
5 | Accept:
6 | - '*/*'
7 | Accept-Encoding:
8 | - gzip, deflate, zstd
9 | Connection:
10 | - keep-alive
11 | Content-Length:
12 | - '16'
13 | User-Agent:
14 | - Python-SDK/0.3.41
15 | content-type:
16 | - application/json
17 | method: POST
18 | uri: https://staging-api.notdiamond.ai/v2/preferences/userPreferenceCreate
19 | response:
20 | body:
21 | string: '{"preference_id":"bcf5f5c4-1a51-44b4-8054-efe8916e79c4"}'
22 | headers:
23 | CF-RAY:
24 | - 925cfba009f33db8-SOF
25 | Connection:
26 | - keep-alive
27 | Content-Type:
28 | - application/json
29 | Date:
30 | - Tue, 25 Mar 2025 08:19:22 GMT
31 | Server:
32 | - cloudflare
33 | Transfer-Encoding:
34 | - chunked
35 | alt-svc:
36 | - h3=":443"; ma=86400
37 | cf-cache-status:
38 | - DYNAMIC
39 | content-length:
40 | - '56'
41 | rndr-id:
42 | - 73c5294b-6a29-488c
43 | vary:
44 | - Accept-Encoding
45 | x-render-origin-server:
46 | - uvicorn
47 | status:
48 | code: 200
49 | message: OK
50 | version: 1
51 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_preference_id[_NDClientTarget.ROUTER-_NDRouterClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"name": "test"}'
4 | headers:
5 | Accept:
6 | - '*/*'
7 | Accept-Encoding:
8 | - gzip, deflate, zstd
9 | Connection:
10 | - keep-alive
11 | Content-Length:
12 | - '16'
13 | User-Agent:
14 | - Python-SDK/0.3.41
15 | content-type:
16 | - application/json
17 | method: POST
18 | uri: https://staging-api.notdiamond.ai/v2/preferences/userPreferenceCreate
19 | response:
20 | body:
21 | string: '{"preference_id":"defe1ede-025b-4e5d-b30e-3f0375fb9be8"}'
22 | headers:
23 | CF-RAY:
24 | - 925cfb9d6816d0da-SOF
25 | Connection:
26 | - keep-alive
27 | Content-Type:
28 | - application/json
29 | Date:
30 | - Tue, 25 Mar 2025 08:19:22 GMT
31 | Server:
32 | - cloudflare
33 | Transfer-Encoding:
34 | - chunked
35 | alt-svc:
36 | - h3=":443"; ma=86400
37 | cf-cache-status:
38 | - DYNAMIC
39 | content-length:
40 | - '56'
41 | rndr-id:
42 | - 02ea0db9-2544-4d9b
43 | vary:
44 | - Accept-Encoding
45 | x-render-origin-server:
46 | - uvicorn
47 | status:
48 | code: 200
49 | message: OK
50 | version: 1
51 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_llm/Test_OpenAI_style_input.test_openai_style_input_model_select[_NDClientTarget.INVOKER-_NDInvokerClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Hello, what''s your name?"},
4 | {"role": "assistant", "content": "My name is Isaac Asimov"}, {"role": "user",
5 | "content": "And what do you do?"}], "llm_providers": [{"provider": "openai",
6 | "model": "gpt-3.5-turbo", "is_custom": false, "context_length": null, "input_price":
7 | null, "output_price": null, "latency": null}], "metric": "accuracy", "max_model_depth":
8 | 1, "hash_content": true}'
9 | headers:
10 | Accept:
11 | - '*/*'
12 | Accept-Encoding:
13 | - gzip, deflate, zstd
14 | Connection:
15 | - keep-alive
16 | Content-Length:
17 | - '423'
18 | User-Agent:
19 | - Python-SDK/0.3.41
20 | content-type:
21 | - application/json
22 | method: POST
23 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
24 | response:
25 | body:
26 | string: '{"providers":[{"provider":"openai","model":"gpt-3.5-turbo"}],"session_id":"38bb1226-b663-42e7-bbdd-dbeaec1f2604"}'
27 | headers:
28 | CF-RAY:
29 | - 925cfbd2f8f9c270-VIE
30 | Connection:
31 | - keep-alive
32 | Content-Type:
33 | - application/json
34 | Date:
35 | - Tue, 25 Mar 2025 08:19:31 GMT
36 | Server:
37 | - cloudflare
38 | Transfer-Encoding:
39 | - chunked
40 | alt-svc:
41 | - h3=":443"; ma=86400
42 | cf-cache-status:
43 | - DYNAMIC
44 | content-length:
45 | - '113'
46 | rndr-id:
47 | - ded4ecf8-c43a-4d5c
48 | vary:
49 | - Accept-Encoding
50 | x-render-origin-server:
51 | - uvicorn
52 | status:
53 | code: 200
54 | message: OK
55 | version: 1
56 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/cassettes/test_llm/Test_OpenAI_style_input.test_openai_style_input_model_select[_NDClientTarget.ROUTER-_NDRouterClient].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Hello, what''s your name?"},
4 | {"role": "assistant", "content": "My name is Isaac Asimov"}, {"role": "user",
5 | "content": "And what do you do?"}], "llm_providers": [{"provider": "openai",
6 | "model": "gpt-3.5-turbo", "is_custom": false, "context_length": null, "input_price":
7 | null, "output_price": null, "latency": null}], "metric": "accuracy", "max_model_depth":
8 | 1, "hash_content": true}'
9 | headers:
10 | Accept:
11 | - '*/*'
12 | Accept-Encoding:
13 | - gzip, deflate, zstd
14 | Connection:
15 | - keep-alive
16 | Content-Length:
17 | - '423'
18 | User-Agent:
19 | - Python-SDK/0.3.41
20 | content-type:
21 | - application/json
22 | method: POST
23 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
24 | response:
25 | body:
26 | string: '{"providers":[{"provider":"openai","model":"gpt-3.5-turbo"}],"session_id":"858a6f9d-bd1d-4a0b-9991-fab4c05bb797"}'
27 | headers:
28 | CF-RAY:
29 | - 925cfbcec8a75b13-VIE
30 | Connection:
31 | - keep-alive
32 | Content-Type:
33 | - application/json
34 | Date:
35 | - Tue, 25 Mar 2025 08:19:30 GMT
36 | Server:
37 | - cloudflare
38 | Transfer-Encoding:
39 | - chunked
40 | alt-svc:
41 | - h3=":443"; ma=86400
42 | cf-cache-status:
43 | - DYNAMIC
44 | content-length:
45 | - '113'
46 | rndr-id:
47 | - 975085bd-655f-4f7c
48 | vary:
49 | - Accept-Encoding
50 | x-render-origin-server:
51 | - uvicorn
52 | status:
53 | code: 200
54 | message: OK
55 | version: 1
56 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/test_callbacks.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from notdiamond.llms.client import _NDClientTarget, _ndllm_factory
4 | from notdiamond.llms.providers import NDLLMProviders
5 |
6 |
7 | @pytest.mark.vcr
8 | @pytest.mark.parametrize(
9 | "nd_llm_cls",
10 | [
11 | (_ndllm_factory(_NDClientTarget.ROUTER)),
12 | (_ndllm_factory(_NDClientTarget.INVOKER)),
13 | ],
14 | )
15 | def test_model_select_callback(llm_base_callback_handler, nd_llm_cls):
16 | """
17 | This tests that the on_model_select method is called when a model is selected.
18 | """
19 |
20 | nd_llm = nd_llm_cls(
21 | llm_configs=[
22 | NDLLMProviders.GPT_4_TURBO_PREVIEW,
23 | NDLLMProviders.GPT_3_5_TURBO,
24 | ],
25 | callbacks=[llm_base_callback_handler],
26 | )
27 | nd_llm.model_select(
28 | messages=[{"role": "user", "content": "Hello, what's your name?"}]
29 | )
30 | assert llm_base_callback_handler.on_model_select_called
31 |
32 |
33 | @pytest.mark.vcr
34 | def test_latency_tracking_callback(llm_base_callback_handler, nd_invoker_cls):
35 | """
36 | This tests that latency tracking is enabled and the on_latency_tracking method is called.
37 | """
38 | nd_llm = nd_invoker_cls(
39 | llm_configs=[
40 | NDLLMProviders.GPT_4_TURBO_PREVIEW,
41 | NDLLMProviders.GPT_3_5_TURBO,
42 | ],
43 | callbacks=[llm_base_callback_handler],
44 | latency_tracking=True,
45 | )
46 | nd_llm.invoke(
47 | messages=[{"role": "user", "content": "Hello, what's your name?"}]
48 | )
49 | assert llm_base_callback_handler.on_latency_tracking_called
50 |
51 |
52 | @pytest.mark.vcr
53 | def test_llm_start_callback(llm_base_callback_handler, nd_invoker_cls):
54 | """
55 | This tests that callback handler is passed along correctly to
56 | the langchain LLM class, by checking if the on_llm_start method is called.
57 | """
58 | nd_llm = nd_invoker_cls(
59 | llm_configs=[
60 | NDLLMProviders.GPT_4_TURBO_PREVIEW,
61 | NDLLMProviders.GPT_3_5_TURBO,
62 | ],
63 | callbacks=[llm_base_callback_handler],
64 | )
65 | nd_llm.invoke(messages=[{"role": "user", "content": "Tell me a joke."}])
66 | assert llm_base_callback_handler.on_llm_start_called
67 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/test_embedding_config.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from notdiamond.exceptions import UnsupportedEmbeddingProvider
4 | from notdiamond.llms.config import EmbeddingConfig
5 |
6 |
7 | def test_supported_embedding_provider():
8 | config = EmbeddingConfig(provider="openai", model="text-embedding-3-small")
9 | assert config.provider == "openai"
10 | assert config.model == "text-embedding-3-small"
11 |
12 | config = EmbeddingConfig(provider="mistral", model="mistral-embed")
13 | assert config.provider == "mistral"
14 | assert config.model == "mistral-embed"
15 |
16 | config = EmbeddingConfig(provider="cohere", model="embed-english-v3.0")
17 | assert config.provider == "cohere"
18 | assert config.model == "embed-english-v3.0"
19 |
20 |
21 | def test_unsupported_embedding_model():
22 | with pytest.raises(UnsupportedEmbeddingProvider):
23 | EmbeddingConfig(provider="openai", model="gpt-71")
24 |
25 |
26 | def test_config_from_string():
27 | config = EmbeddingConfig.from_string("openai/text-embedding-3-small")
28 | assert config.provider == "openai"
29 | assert config.model == "text-embedding-3-small"
30 |
--------------------------------------------------------------------------------
/tests/test_components/test_llms/test_provider.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 | import pytest
4 |
5 | from notdiamond import LLMConfig
6 | from notdiamond.exceptions import UnsupportedLLMProvider
7 |
8 | sys.path.append("../")
9 |
10 |
11 | def test_supported_llm_provider():
12 | openai = LLMConfig(provider="openai", model="gpt-3.5-turbo")
13 | assert openai.provider == "openai"
14 | assert openai.model == "gpt-3.5-turbo"
15 |
16 | openai = LLMConfig(provider="anthropic", model="claude-2.1")
17 | assert openai.provider == "anthropic"
18 | assert openai.model == "claude-2.1"
19 |
20 | openai = LLMConfig(provider="google", model="gemini-pro")
21 | assert openai.provider == "google"
22 | assert openai.model == "gemini-pro"
23 |
24 |
25 | def test_unsupported_model():
26 | with pytest.raises(UnsupportedLLMProvider):
27 | LLMConfig(provider="openai", model="gpt-71")
28 |
29 |
30 | def test_prepare_for_request():
31 | llm_provider = LLMConfig(
32 | provider="openai",
33 | model="gpt-3.5-turbo",
34 | context_length=1,
35 | input_price=0.1,
36 | output_price=0.4,
37 | latency=100,
38 | )
39 | request = llm_provider.prepare_for_request()
40 | assert request == {
41 | "provider": llm_provider.provider,
42 | "model": llm_provider.model,
43 | "is_custom": llm_provider.is_custom,
44 | "context_length": llm_provider.context_length,
45 | "input_price": llm_provider.input_price,
46 | "output_price": llm_provider.output_price,
47 | "latency": llm_provider.latency,
48 | }
49 |
50 |
51 | def test_existing_openrouter_model():
52 | llm_provider = LLMConfig(provider="mistral", model="mistral-large-latest")
53 | assert llm_provider.openrouter_model == "mistralai/mistral-large"
54 |
55 | llm_provider = LLMConfig(
56 | provider="anthropic", model="claude-3-opus-20240229"
57 | )
58 | assert llm_provider.openrouter_model == "anthropic/claude-3-opus"
59 |
60 |
61 | def test_not_existing_openrouter_model():
62 | llm_provider = LLMConfig(provider="openai", model="gpt-4-0125-preview")
63 | result = llm_provider.openrouter_model
64 | assert result is None
65 |
--------------------------------------------------------------------------------
/tests/test_components/test_utils.py:
--------------------------------------------------------------------------------
1 | """Tests for utility functions in _utils.py"""
2 |
3 | import pytest
4 |
5 | from notdiamond._utils import token_counter
6 |
7 |
8 | class TestTokenCounter:
9 | """Test class for token_counter function"""
10 |
11 | def test_token_counter_with_text(self):
12 | """Test token_counter with text input"""
13 | text = "This is a simple test sentence."
14 | count = token_counter(model="gpt-4", text=text)
15 | assert isinstance(count, int)
16 | assert count > 0
17 | assert count <= len(text) # Tokens should be fewer than characters
18 |
19 | def test_token_counter_with_messages(self):
20 | """Test token_counter with messages input"""
21 | messages = [
22 | {"role": "user", "content": "Hello, how are you?"},
23 | {"role": "assistant", "content": "I'm doing well, thank you for asking!"}
24 | ]
25 | count = token_counter(model="gpt-3.5-turbo", messages=messages)
26 | assert isinstance(count, int)
27 | assert count > 0
28 |
29 | # Verify that computing tokens from text and from messages gives consistent results
30 | combined_text = "Hello, how are you?I'm doing well, thank you for asking!"
31 | text_count = token_counter(model="gpt-3.5-turbo", text=combined_text)
32 | assert count >= text_count # Messages might add a few tokens for role markers
33 |
34 | def test_token_counter_different_models(self):
35 | """Test token_counter with different models"""
36 | text = "This is a test of token counting across different models."
37 |
38 | gpt4_count = token_counter(model="gpt-4", text=text)
39 | gpt35_count = token_counter(model="gpt-3.5-turbo", text=text)
40 |
41 | # Token counts should be similar for same text across different models
42 | assert abs(gpt4_count - gpt35_count) <= 2
43 |
44 | def test_token_counter_fallback(self):
45 | """Test token_counter fallback mechanism with unknown model"""
46 | text = "Testing the fallback mechanism with an unknown model."
47 |
48 | # Test with a model name that won't have a specific tokenizer
49 | count = token_counter(model="unknown-model", text=text)
50 | assert isinstance(count, int)
51 | assert count > 0
52 |
53 | # Compare with a known model's count
54 | known_model_count = token_counter(model="gpt-4", text=text)
55 | # The counts should be reasonably similar
56 | assert abs(count - known_model_count) <= 5
57 |
58 | def test_token_counter_error_handling(self):
59 | """Test token_counter error handling"""
60 | # Test with neither text nor messages
61 | with pytest.raises(ValueError):
62 | token_counter(model="gpt-4")
--------------------------------------------------------------------------------
/tests/test_documentation/cassettes/test_function_calling/test_function_calling_via_rest_api.yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "system", "content": "You are a mathematician with
4 | tools at your disposal."}, {"role": "assistant", "content": "How can I help
5 | you today?"}, {"role": "user", "content": "What is 234 + 82234?"}], "llm_providers":
6 | [{"provider": "openai", "model": "gpt-4-1106-preview"}, {"provider": "anthropic",
7 | "model": "claude-3-opus-20240229"}], "tools": [{"type": "function", "function":
8 | {"name": "add", "description": "Adds a and b.", "parameters": {"type": "object",
9 | "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}, "required":
10 | ["a", "b"]}}}, {"type": "function", "function": {"name": "multiply", "description":
11 | "Multiplies a and b.", "parameters": {"type": "object", "properties": {"a":
12 | {"type": "integer"}, "b": {"type": "integer"}}, "required": ["a", "b"]}}}]}'
13 | headers:
14 | Accept-Encoding:
15 | - gzip, deflate, zstd
16 | Connection:
17 | - keep-alive
18 | Content-Length:
19 | - '793'
20 | User-Agent:
21 | - python-requests/2.32.3
22 | accept:
23 | - application/json
24 | content-type:
25 | - application/json
26 | method: POST
27 | uri: https://staging-api.notdiamond.ai/v2/optimizer/hashModelSelect
28 | response:
29 | body:
30 | string: '{"providers":[{"provider":"openai","model":"gpt-4-1106-preview"}],"session_id":"1c01fac2-0bba-4d25-b9ec-ecc035e73dc4"}'
31 | headers:
32 | CF-RAY:
33 | - 925d1ba3ce02d0ec-SOF
34 | Connection:
35 | - keep-alive
36 | Content-Type:
37 | - application/json
38 | Date:
39 | - Tue, 25 Mar 2025 08:41:14 GMT
40 | Server:
41 | - cloudflare
42 | Transfer-Encoding:
43 | - chunked
44 | alt-svc:
45 | - h3=":443"; ma=86400
46 | cf-cache-status:
47 | - DYNAMIC
48 | content-length:
49 | - '118'
50 | rndr-id:
51 | - f828ef91-15d2-4403
52 | vary:
53 | - Accept-Encoding
54 | x-render-origin-server:
55 | - uvicorn
56 | status:
57 | code: 200
58 | message: OK
59 | version: 1
60 |
--------------------------------------------------------------------------------
/tests/test_documentation/cassettes/test_getting_started/test_model_select.yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "system", "content": "You are a world class software
4 | developer."}, {"role": "user", "content": "Write a merge sort in Python"}],
5 | "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo", "is_custom":
6 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
7 | null}, {"provider": "openai", "model": "gpt-4", "is_custom": false, "context_length":
8 | null, "input_price": null, "output_price": null, "latency": null}, {"provider":
9 | "openai", "model": "gpt-4-1106-preview", "is_custom": false, "context_length":
10 | null, "input_price": null, "output_price": null, "latency": null}, {"provider":
11 | "openai", "model": "gpt-4-turbo-preview", "is_custom": false, "context_length":
12 | null, "input_price": null, "output_price": null, "latency": null}, {"provider":
13 | "anthropic", "model": "claude-3-haiku-20240307", "is_custom": false, "context_length":
14 | null, "input_price": null, "output_price": null, "latency": null}, {"provider":
15 | "anthropic", "model": "claude-3-sonnet-20240229", "is_custom": false, "context_length":
16 | null, "input_price": null, "output_price": null, "latency": null}, {"provider":
17 | "anthropic", "model": "claude-3-opus-20240229", "is_custom": false, "context_length":
18 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
19 | "accuracy", "max_model_depth": 7, "hash_content": false, "tradeoff": "cost"}'
20 | headers:
21 | Accept:
22 | - '*/*'
23 | Accept-Encoding:
24 | - gzip, deflate, zstd
25 | Connection:
26 | - keep-alive
27 | Content-Length:
28 | - '1377'
29 | User-Agent:
30 | - Python-SDK/0.3.41
31 | content-type:
32 | - application/json
33 | method: POST
34 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
35 | response:
36 | body:
37 | string: '{"providers":[{"provider":"openai","model":"gpt-4-1106-preview"}],"session_id":"68a6cf92-0acc-4608-8ddd-4b8dadae3aec"}'
38 | headers:
39 | CF-RAY:
40 | - 925cfdab7adec30a-VIE
41 | Connection:
42 | - keep-alive
43 | Content-Type:
44 | - application/json
45 | Date:
46 | - Tue, 25 Mar 2025 08:20:46 GMT
47 | Server:
48 | - cloudflare
49 | Transfer-Encoding:
50 | - chunked
51 | alt-svc:
52 | - h3=":443"; ma=86400
53 | cf-cache-status:
54 | - DYNAMIC
55 | content-length:
56 | - '118'
57 | rndr-id:
58 | - 709cef38-95a6-4614
59 | vary:
60 | - Accept-Encoding
61 | x-render-origin-server:
62 | - uvicorn
63 | status:
64 | code: 200
65 | message: OK
66 | version: 1
67 |
--------------------------------------------------------------------------------
/tests/test_documentation/test_fallback_and_custom.py:
--------------------------------------------------------------------------------
1 | import pytest
2 | from openai import OpenAI
3 |
4 | from notdiamond import settings
5 | from notdiamond.llms.client import NotDiamond
6 |
7 |
8 | @pytest.mark.vcr
9 | def test_fallback_model():
10 | llm_configs = [
11 | "openai/gpt-3.5-turbo",
12 | "openai/gpt-4",
13 | "openai/gpt-4-1106-preview",
14 | "openai/gpt-4-turbo-preview",
15 | "anthropic/claude-2.1",
16 | "anthropic/claude-3-sonnet-20240229",
17 | "anthropic/claude-3-opus-20240229",
18 | "google/gemini-pro",
19 | ]
20 |
21 | # Setting a fallback model
22 | NotDiamond(
23 | llm_configs=llm_configs,
24 | default="openai/gpt-4-turbo-preview", # The model from llm_configs you want to fallback to
25 | )
26 |
27 |
28 | @pytest.mark.vcr
29 | def test_set_max_model_depth():
30 | llm_configs = [
31 | "openai/gpt-3.5-turbo",
32 | "openai/gpt-4",
33 | "openai/gpt-4-1106-preview",
34 | "openai/gpt-4-turbo-preview",
35 | "anthropic/claude-2.1",
36 | "anthropic/claude-3-sonnet-20240229",
37 | "anthropic/claude-3-opus-20240229",
38 | "google/gemini-pro",
39 | ]
40 |
41 | # Setting recommendation depth
42 | NotDiamond(
43 | llm_configs=llm_configs,
44 | max_model_depth=3, # The maximum depth in the recommendation ranking to consider
45 | )
46 |
47 |
48 | @pytest.mark.vcr
49 | def test_custom_logic():
50 | # Define the string that will be routed to the best LLM
51 | prompt = "You are a world class software developer. Write a merge sort in Python."
52 |
53 | # Define the available LLMs you'd like to route between
54 | llm_configs = [
55 | "openai/gpt-3.5-turbo",
56 | ]
57 |
58 | # Create the NDLLM object -> like a 'meta-LLM' combining all of the specified models
59 | nd_llm = NotDiamond(
60 | llm_configs=llm_configs,
61 | tradeoff="cost",
62 | ) # Define preferences
63 |
64 | session_id, provider = nd_llm.model_select(messages=prompt)
65 |
66 | print(session_id)
67 | print(provider)
68 |
69 | client = OpenAI(api_key=settings.OPENAI_API_KEY)
70 |
71 | max_retries = 3
72 |
73 | if provider.model == "gpt-3.5-turbo":
74 | for _ in range(max_retries):
75 | try:
76 | chat_completion = client.chat.completions.create(
77 | messages=[
78 | {
79 | "role": "user",
80 | "content": prompt,
81 | }
82 | ],
83 | model="gpt-3.5-turbo",
84 | )
85 | return chat_completion.choices[0].message.content
86 | except Exception as e:
87 | print(e)
88 | continue
89 |
--------------------------------------------------------------------------------
/tests/test_documentation/test_langchain.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from notdiamond.llms.client import NotDiamond
4 |
5 |
6 | @pytest.mark.vcr
7 | def test_streaming():
8 | messages = [
9 | {
10 | "role": "system",
11 | "content": "You are a world class software developer.",
12 | },
13 | {"role": "user", "content": "Write merge sort in Python."},
14 | ]
15 |
16 | chat = NotDiamond(
17 | llm_configs=[
18 | "openai/gpt-3.5-turbo",
19 | "openai/gpt-4",
20 | "anthropic/claude-2.1",
21 | "google/gemini-pro",
22 | ]
23 | )
24 |
25 | for chunk in chat.stream(messages):
26 | print(chunk.content, end="", flush=True)
27 |
--------------------------------------------------------------------------------
/tests/test_documentation/test_openrouter.py:
--------------------------------------------------------------------------------
1 | import json
2 | import os
3 |
4 | import pytest
5 | import requests
6 | from dotenv import load_dotenv
7 |
8 | from notdiamond.llms.client import NotDiamond
9 |
10 | load_dotenv(os.getcwd() + "/.env")
11 |
12 |
13 | @pytest.mark.vcr
14 | def test_openrouter_integration():
15 | openrouter_api_key = os.getenv("OPENROUTER_API_KEY", default="")
16 | messages = [
17 | {
18 | "role": "system",
19 | "content": "You are a world class software developer.",
20 | },
21 | {"role": "user", "content": "Write a merge sort in Python"},
22 | ]
23 |
24 | llm_configs = [
25 | "openai/gpt-3.5-turbo",
26 | "openai/gpt-4",
27 | "openai/gpt-4-1106-preview",
28 | "openai/gpt-4-turbo-preview",
29 | "anthropic/claude-3-haiku-20240307",
30 | "anthropic/claude-3-sonnet-20240229",
31 | "anthropic/claude-3-opus-20240229",
32 | ]
33 |
34 | nd_llm = NotDiamond(llm_configs=llm_configs)
35 | session_id, provider = nd_llm.model_select(messages=messages)
36 |
37 | response = requests.post(
38 | url="https://openrouter.ai/api/v1/chat/completions",
39 | headers={
40 | "Authorization": f"Bearer {openrouter_api_key}",
41 | },
42 | data=json.dumps(
43 | {"model": provider.openrouter_model, "messages": messages}
44 | ),
45 | )
46 |
47 | print(response)
48 |
--------------------------------------------------------------------------------
/tests/test_documentation/test_personalization.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from notdiamond.llms.client import NotDiamond
4 | from notdiamond.metrics.metric import Metric
5 |
6 |
7 | @pytest.mark.vcr
8 | def test_personalization():
9 | prompt = [
10 | {
11 | "role": "user",
12 | "content": "You are a world class software developer. Write a merge sort in Python.",
13 | }
14 | ]
15 |
16 | llm_configs = [
17 | "openai/gpt-3.5-turbo",
18 | "openai/gpt-4",
19 | "openai/gpt-4-1106-preview",
20 | "openai/gpt-4-turbo-preview",
21 | "anthropic/claude-2.1",
22 | "anthropic/claude-3-sonnet-20240229",
23 | "anthropic/claude-3-opus-20240229",
24 | "google/gemini-pro",
25 | ]
26 |
27 | nd_llm = NotDiamond(
28 | llm_configs=llm_configs,
29 | preference_id="5c5af286-b715-4d8b-8cf9-151230ef96a3",
30 | )
31 | metric = Metric("accuracy")
32 |
33 | result, session_id, provider = nd_llm.invoke(prompt, metric=metric)
34 |
35 | # Application logic...
36 | # Let's assume the result from the LLM achieved the outcome you're looking for!
37 |
38 | score = metric.feedback(
39 | session_id=session_id, llm_config=provider, value=1
40 | )
41 | print(score)
42 |
--------------------------------------------------------------------------------
/tests/test_documentation/test_structured_output.py:
--------------------------------------------------------------------------------
1 | from json import JSONDecoder
2 |
3 | import pytest
4 | from langchain_core.exceptions import OutputParserException
5 | from pydantic import BaseModel, Field
6 |
7 | from notdiamond.llms.client import NotDiamond
8 |
9 |
10 | @pytest.mark.vcr
11 | def test_structured_output():
12 | class LanguageChoice(BaseModel):
13 | language: str = Field(
14 | description="The programming language of choice."
15 | )
16 | reason: str = Field(
17 | description="The reason to pick the programming language."
18 | )
19 |
20 | messages = [
21 | {
22 | "role": "system",
23 | "content": "You are a world class software developer.",
24 | },
25 | {
26 | "role": "user",
27 | "content": "What language would you suggest for developing a web application?",
28 | },
29 | ]
30 |
31 | llm_configs = [
32 | "openai/gpt-3.5-turbo",
33 | "openai/gpt-4",
34 | "openai/gpt-4-1106-preview",
35 | "openai/gpt-4-turbo-preview",
36 | "anthropic/claude-3-opus-20240229",
37 | ]
38 |
39 | nd_llm = NotDiamond(llm_configs=llm_configs)
40 |
41 | def extract_json_objects(text, decoder=JSONDecoder()):
42 | """Find JSON objects in text, and yield the decoded JSON data
43 |
44 | Does not attempt to look for JSON arrays, text, or other JSON types outside
45 | of a parent JSON object.
46 |
47 | """
48 | pos = 0
49 | while True:
50 | match = text.find("{", pos)
51 | if match == -1:
52 | break
53 | try:
54 | result, index = decoder.raw_decode(text[match:])
55 | yield result
56 | pos = match + index
57 | except ValueError:
58 | pos = match + 1
59 |
60 | try:
61 | result, session_id, _ = nd_llm.invoke(
62 | messages=messages, response_model=LanguageChoice
63 | )
64 | except OutputParserException:
65 | result, session_id, _ = nd_llm.invoke(messages=messages)
66 | json_obj = list(extract_json_objects(result.content))
67 | assert len(json_obj) == 1
68 | json_response = json_obj[0]
69 | assert len(json_response) == 2
70 | assert "language" in json_response
71 | assert "reason" in json_response
72 |
73 | print(result)
74 |
75 |
76 | @pytest.mark.vcr
77 | def test_structured_output_streaming():
78 | class LanguageChoice(BaseModel):
79 | language: str = Field(
80 | description="The programming language of choice."
81 | )
82 | reason: str = Field(
83 | description="The reason to pick the programming language."
84 | )
85 |
86 | messages = [
87 | {
88 | "role": "system",
89 | "content": "You are a world class software developer.",
90 | },
91 | {
92 | "role": "user",
93 | "content": "What language would you suggest for developing a web application?",
94 | },
95 | ]
96 |
97 | llm_configs = [
98 | "openai/gpt-3.5-turbo",
99 | "openai/gpt-4",
100 | "openai/gpt-4-1106-preview",
101 | "openai/gpt-4-turbo-preview",
102 | "anthropic/claude-3-opus-20240229",
103 | ]
104 |
105 | nd_llm = NotDiamond(llm_configs=llm_configs)
106 |
107 | for chunk in nd_llm.stream(
108 | messages=messages, response_model=LanguageChoice
109 | ):
110 | print(chunk)
111 |
--------------------------------------------------------------------------------
/tests/test_llm_calls/test_cohere.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from notdiamond.llms.client import NotDiamond
4 | from notdiamond.llms.providers import NDLLMProviders
5 |
6 |
7 | @pytest.mark.vcr
8 | @pytest.mark.longrun
9 | class Test_Cohere:
10 | def test_cohere_command_r_plus_with_tool_calling(self, tools_fixture):
11 | provider = NDLLMProviders.COMMAND_R_PLUS
12 | provider.kwargs = {"max_tokens": 200}
13 | nd_llm = NotDiamond(
14 | llm_configs=[provider], latency_tracking=False, hash_content=True
15 | )
16 | nd_llm = nd_llm.bind_tools(tools_fixture)
17 | result, session_id, _ = nd_llm.invoke(
18 | [{"role": "user", "content": "How much is 3 + 5?"}]
19 | )
20 |
21 | assert len(result.tool_calls) == 1
22 | assert result.tool_calls[0]["name"] == "add_fct"
23 |
24 | def test_cohere_command_r_plus_response_model(self, response_model):
25 | provider = NDLLMProviders.COMMAND_R_PLUS
26 | provider.kwargs = {"max_tokens": 200}
27 | nd_llm = NotDiamond(
28 | llm_configs=[provider], latency_tracking=False, hash_content=True
29 | )
30 | result, _, _ = nd_llm.invoke(
31 | [{"role": "user", "content": "Tell me a joke"}],
32 | response_model=response_model,
33 | )
34 |
35 | assert isinstance(result, response_model)
36 | assert result.setup
37 | assert result.punchline
38 |
39 | def test_cohere_command_r_with_tool_calling(self, tools_fixture):
40 | provider = NDLLMProviders.COMMAND_R
41 | provider.kwargs = {"max_tokens": 200}
42 | nd_llm = NotDiamond(
43 | llm_configs=[provider], latency_tracking=False, hash_content=True
44 | )
45 | nd_llm = nd_llm.bind_tools(tools_fixture)
46 | result, session_id, _ = nd_llm.invoke(
47 | [{"role": "user", "content": "How much is 3 + 5?"}]
48 | )
49 |
50 | assert len(result.tool_calls) == 1
51 | assert result.tool_calls[0]["name"] == "add_fct"
52 |
53 | def test_cohere_command_r_response_model(self, response_model):
54 | provider = NDLLMProviders.COMMAND_R
55 | provider.kwargs = {"max_tokens": 200}
56 | nd_llm = NotDiamond(
57 | llm_configs=[provider], latency_tracking=False, hash_content=True
58 | )
59 | result, _, _ = nd_llm.invoke(
60 | [{"role": "user", "content": "Tell me a joke"}],
61 | response_model=response_model,
62 | )
63 |
64 | assert isinstance(result, response_model)
65 | assert result.setup
66 | assert result.punchline
67 |
--------------------------------------------------------------------------------
/tests/test_llm_calls/test_openai_o1.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from notdiamond.llms.client import NotDiamond
4 | from notdiamond.llms.providers import NDLLMProviders
5 |
6 | test_providers = [
7 | provider
8 | for provider in NDLLMProviders
9 | if provider.provider == "openai" and provider.model[:2] == "o1"
10 | ]
11 |
12 |
13 | @pytest.mark.vcr
14 | @pytest.mark.longrun
15 | @pytest.mark.parametrize("provider", test_providers)
16 | def test_o1_with_system_prompt(provider):
17 | nd_llm = NotDiamond(
18 | llm_configs=[provider], latency_tracking=False, hash_content=True
19 | )
20 | result, session_id, _ = nd_llm.invoke(
21 | [
22 | {"role": "system", "content": "You are a funny AI"},
23 | {"role": "user", "content": "Tell me a joke"},
24 | ],
25 | )
26 |
27 | assert session_id != "NO-SESSION-ID"
28 | assert len(result.content) > 0
29 |
--------------------------------------------------------------------------------
/tests/test_llm_calls/test_perplexity.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from notdiamond.llms.client import NotDiamond
4 | from notdiamond.llms.providers import NDLLMProviders
5 | from notdiamond.metrics.metric import Metric
6 |
7 |
8 | @pytest.mark.vcr
9 | @pytest.mark.longrun
10 | class Test_Perplexity_LLMs:
11 | def test_sonar(self):
12 | provider = NDLLMProviders.SONAR
13 | provider.kwargs = {"max_tokens": 10}
14 | nd_llm = NotDiamond(
15 | llm_configs=[provider],
16 | latency_tracking=False,
17 | hash_content=False,
18 | )
19 | result, session_id, _ = nd_llm.invoke(
20 | messages=[{"role": "user", "content": "How much is 3 + 5?"}],
21 | metric=Metric("accuracy"),
22 | )
23 |
24 | assert session_id != "NO-SESSION-ID"
25 | assert len(result.content) > 0
26 |
--------------------------------------------------------------------------------
/tests/test_llm_calls/test_replicate.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from notdiamond.llms.client import NotDiamond
4 | from notdiamond.llms.providers import NDLLMProviders
5 | from notdiamond.metrics.metric import Metric
6 |
7 |
8 | @pytest.mark.vcr
9 | @pytest.mark.longrun
10 | @pytest.mark.skip("Currently failing, need to debug")
11 | class Test_Replicate_LLMs:
12 | def test_mistral_7b_instruct_v02(self):
13 | provider = NDLLMProviders.REPLICATE_MISTRAL_7B_INSTRUCT_V0_2
14 | provider.kwargs = {"max_tokens": 10}
15 | nd_llm = NotDiamond(
16 | llm_configs=[provider], latency_tracking=False, hash_content=True
17 | )
18 | result, session_id, _ = nd_llm.invoke(
19 | messages=[{"role": "user", "content": "How much is 3 + 5?"}],
20 | metric=Metric("accuracy"),
21 | )
22 |
23 | assert session_id != "NO-SESSION-ID"
24 | assert len(result.content) > 0
25 |
26 | def test_mixtral_8x7b_instruct_v01(self):
27 | provider = NDLLMProviders.REPLICATE_MIXTRAL_8X7B_INSTRUCT_V0_1
28 | provider.kwargs = {"max_tokens": 10}
29 | nd_llm = NotDiamond(
30 | llm_configs=[provider], latency_tracking=False, hash_content=True
31 | )
32 | result, session_id, _ = nd_llm.invoke(
33 | messages=[{"role": "user", "content": "How much is 3 + 5?"}],
34 | metric=Metric("accuracy"),
35 | )
36 |
37 | assert session_id != "NO-SESSION-ID"
38 | assert len(result.content) > 0
39 |
40 | def test_llama_3_70b_instruct(self):
41 | provider = NDLLMProviders.REPLICATE_META_LLAMA_3_70B_INSTRUCT
42 | provider.kwargs = {"max_tokens": 10}
43 | nd_llm = NotDiamond(
44 | llm_configs=[provider],
45 | latency_tracking=False,
46 | hash_content=False,
47 | )
48 | result, session_id, _ = nd_llm.invoke(
49 | messages=[{"role": "user", "content": "How much is 3 + 5?"}],
50 | metric=Metric("accuracy"),
51 | )
52 |
53 | assert session_id != "NO-SESSION-ID"
54 | assert len(result.content) > 0
55 |
56 | def test_llama_3_8b_instruct_with_prompt_template(self):
57 | provider = NDLLMProviders.REPLICATE_META_LLAMA_3_8B_INSTRUCT
58 | provider.kwargs = {"max_tokens": 10}
59 | nd_llm = NotDiamond(
60 | llm_configs=[provider],
61 | latency_tracking=False,
62 | hash_content=False,
63 | )
64 | result, session_id, _ = nd_llm.invoke(
65 | messages=[{"role": "user", "content": "How much is 3 + 5?"}],
66 | metric=Metric("accuracy"),
67 | )
68 |
69 | assert session_id != "NO-SESSION-ID"
70 | assert len(result.content) > 0
71 |
72 | def test_llama_3_1_405b_instruct(self):
73 | provider = NDLLMProviders.REPLICATE_META_LLAMA_3_1_405B_INSTRUCT
74 | provider.kwargs = {"max_tokens": 10}
75 | nd_llm = NotDiamond(
76 | llm_configs=[provider],
77 | latency_tracking=False,
78 | hash_content=False,
79 | )
80 | result, session_id, _ = nd_llm.invoke(
81 | messages=[{"role": "user", "content": "How much is 3 + 5?"}],
82 | metric=Metric("accuracy"),
83 | )
84 |
85 | assert session_id != "NO-SESSION-ID"
86 | assert len(result.content) > 0
87 |
--------------------------------------------------------------------------------
/tests/test_toolkit/cassettes/test_retry/test_retry_wrapper[anthropic].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "Hello, how
4 | are you?"}], "model": "claude-3-5-haiku-20241022"}'
5 | headers:
6 | accept:
7 | - application/json
8 | accept-encoding:
9 | - gzip, deflate
10 | anthropic-version:
11 | - '2023-06-01'
12 | connection:
13 | - keep-alive
14 | content-length:
15 | - '124'
16 | content-type:
17 | - application/json
18 | host:
19 | - api.anthropic.com
20 | user-agent:
21 | - Anthropic/Python 0.39.0
22 | x-api-key:
23 | - MASKED-API-KEY-FOR-TESTING
24 | x-stainless-arch:
25 | - arm64
26 | x-stainless-async:
27 | - 'false'
28 | x-stainless-lang:
29 | - python
30 | x-stainless-os:
31 | - MacOS
32 | x-stainless-package-version:
33 | - 0.39.0
34 | x-stainless-retry-count:
35 | - '0'
36 | x-stainless-runtime:
37 | - CPython
38 | x-stainless-runtime-version:
39 | - 3.11.10
40 | method: POST
41 | uri: https://api.anthropic.com/v1/messages
42 | response:
43 | body:
44 | string: '{"id":"msg_017E8Qpatk7sjidTqfJiRiru","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''m
45 | doing well, thank you for asking! How are you today? Is there anything I can
46 | help you with?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":13,"output_tokens":27}}'
47 | headers:
48 | CF-Cache-Status:
49 | - DYNAMIC
50 | CF-RAY:
51 | - 8ed7549aaa9567de-MIA
52 | Connection:
53 | - keep-alive
54 | Content-Type:
55 | - application/json
56 | Date:
57 | - Thu, 05 Dec 2024 22:04:12 GMT
58 | Server:
59 | - cloudflare
60 | Transfer-Encoding:
61 | - chunked
62 | X-Robots-Tag:
63 | - none
64 | anthropic-ratelimit-input-tokens-limit:
65 | - '400000'
66 | anthropic-ratelimit-input-tokens-remaining:
67 | - '400000'
68 | anthropic-ratelimit-input-tokens-reset:
69 | - '2024-12-05T22:04:12Z'
70 | anthropic-ratelimit-output-tokens-limit:
71 | - '80000'
72 | anthropic-ratelimit-output-tokens-remaining:
73 | - '80000'
74 | anthropic-ratelimit-output-tokens-reset:
75 | - '2024-12-05T22:04:12Z'
76 | anthropic-ratelimit-requests-limit:
77 | - '4000'
78 | anthropic-ratelimit-requests-remaining:
79 | - '3999'
80 | anthropic-ratelimit-requests-reset:
81 | - '2024-12-05T22:04:11Z'
82 | anthropic-ratelimit-tokens-limit:
83 | - '480000'
84 | anthropic-ratelimit-tokens-remaining:
85 | - '480000'
86 | anthropic-ratelimit-tokens-reset:
87 | - '2024-12-05T22:04:12Z'
88 | content-length:
89 | - '335'
90 | request-id:
91 | - req_01Kvba1zUfqLadADM3McoKuf
92 | via:
93 | - 1.1 google
94 | status:
95 | code: 200
96 | message: OK
97 | version: 1
98 |
--------------------------------------------------------------------------------
/tests/test_toolkit/cassettes/test_retry/test_retry_wrapper[openai].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model":
4 | "gpt-4o-mini"}'
5 | headers:
6 | accept:
7 | - application/json
8 | accept-encoding:
9 | - gzip, deflate
10 | connection:
11 | - keep-alive
12 | content-length:
13 | - '90'
14 | content-type:
15 | - application/json
16 | host:
17 | - api.openai.com
18 | user-agent:
19 | - OpenAI/Python 1.55.0
20 | x-stainless-arch:
21 | - arm64
22 | x-stainless-async:
23 | - 'false'
24 | x-stainless-lang:
25 | - python
26 | x-stainless-os:
27 | - MacOS
28 | x-stainless-package-version:
29 | - 1.55.0
30 | x-stainless-retry-count:
31 | - '0'
32 | x-stainless-runtime:
33 | - CPython
34 | x-stainless-runtime-version:
35 | - 3.11.10
36 | method: POST
37 | uri: https://api.openai.com/v1/chat/completions
38 | response:
39 | body:
40 | string: "{\n \"id\": \"chatcmpl-AbEF8ClecDZCPMUjm7boOu2lFSDVl\",\n \"object\":
41 | \"chat.completion\",\n \"created\": 1733436250,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
42 | \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
43 | \"assistant\",\n \"content\": \"Hello! I'm just a computer program,
44 | so I don't have feelings, but I'm here and ready to help you. How can I assist
45 | you today?\",\n \"refusal\": null\n },\n \"logprobs\": null,\n
46 | \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\":
47 | 13,\n \"completion_tokens\": 30,\n \"total_tokens\": 43,\n \"prompt_tokens_details\":
48 | {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
49 | {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
50 | 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"system_fingerprint\":
51 | \"fp_0705bf87c0\"\n}\n"
52 | headers:
53 | CF-Cache-Status:
54 | - DYNAMIC
55 | CF-RAY:
56 | - 8ed75494abf8db2d-MIA
57 | Connection:
58 | - keep-alive
59 | Content-Type:
60 | - application/json
61 | Date:
62 | - Thu, 05 Dec 2024 22:04:11 GMT
63 | Server:
64 | - cloudflare
65 | Set-Cookie:
66 | - __cf_bm=GA2qg.kK2zm.1O5mc33iZL0OJ9qc4lZs.2E.kwVptfQ-1733436251-1.0.1.1-yG7_72uvTjgx5GWzsLPfFlmwGYhO5HNxBqoU98ExQIswRbZ9mMJq2amr5mcayHgeVKhyfRulb8iUl_umvH54Rg;
67 | path=/; expires=Thu, 05-Dec-24 22:34:11 GMT; domain=.api.openai.com; HttpOnly;
68 | Secure; SameSite=None
69 | - _cfuvid=TwJ5E0YXwwWinJhkalH3gS5WYUKa6w30Fkfh_3x_OaU-1733436251203-0.0.1.1-604800000;
70 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
71 | Transfer-Encoding:
72 | - chunked
73 | X-Content-Type-Options:
74 | - nosniff
75 | access-control-expose-headers:
76 | - X-Request-ID
77 | alt-svc:
78 | - h3=":443"; ma=86400
79 | content-length:
80 | - '876'
81 | openai-organization:
82 | - not-diamond-bdz8cg
83 | openai-processing-ms:
84 | - '776'
85 | openai-version:
86 | - '2020-10-01'
87 | strict-transport-security:
88 | - max-age=31536000; includeSubDomains; preload
89 | x-ratelimit-limit-requests:
90 | - '30000'
91 | x-ratelimit-limit-tokens:
92 | - '150000000'
93 | x-ratelimit-remaining-requests:
94 | - '29999'
95 | x-ratelimit-remaining-tokens:
96 | - '149999978'
97 | x-ratelimit-reset-requests:
98 | - 2ms
99 | x-ratelimit-reset-tokens:
100 | - 0s
101 | x-request-id:
102 | - req_74329deb7921fcd31e0c9ce9b89a0ede
103 | status:
104 | code: 200
105 | message: OK
106 | version: 1
107 |
--------------------------------------------------------------------------------
/tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_async[async-anthropic].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "Hello, how
4 | are you?"}], "model": "claude-3-5-haiku-20241022"}'
5 | headers:
6 | accept:
7 | - application/json
8 | accept-encoding:
9 | - gzip, deflate
10 | anthropic-version:
11 | - '2023-06-01'
12 | connection:
13 | - keep-alive
14 | content-length:
15 | - '124'
16 | content-type:
17 | - application/json
18 | host:
19 | - api.anthropic.com
20 | user-agent:
21 | - AsyncAnthropic/Python 0.39.0
22 | x-api-key:
23 | - MASKED-API-KEY-FOR-TESTING
24 | x-stainless-arch:
25 | - arm64
26 | x-stainless-async:
27 | - async:asyncio
28 | x-stainless-lang:
29 | - python
30 | x-stainless-os:
31 | - MacOS
32 | x-stainless-package-version:
33 | - 0.39.0
34 | x-stainless-retry-count:
35 | - '0'
36 | x-stainless-runtime:
37 | - CPython
38 | x-stainless-runtime-version:
39 | - 3.11.10
40 | method: POST
41 | uri: https://api.anthropic.com/v1/messages
42 | response:
43 | body:
44 | string: '{"id":"msg_01EssLNatxqdSzkXEX1veU71","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''m
45 | doing well, thank you for asking! How are you today?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":13,"output_tokens":18}}'
46 | headers:
47 | CF-Cache-Status:
48 | - DYNAMIC
49 | CF-RAY:
50 | - 8ed754a62dbd3364-MIA
51 | Connection:
52 | - keep-alive
53 | Content-Type:
54 | - application/json
55 | Date:
56 | - Thu, 05 Dec 2024 22:04:13 GMT
57 | Server:
58 | - cloudflare
59 | Transfer-Encoding:
60 | - chunked
61 | X-Robots-Tag:
62 | - none
63 | anthropic-ratelimit-input-tokens-limit:
64 | - '400000'
65 | anthropic-ratelimit-input-tokens-remaining:
66 | - '400000'
67 | anthropic-ratelimit-input-tokens-reset:
68 | - '2024-12-05T22:04:13Z'
69 | anthropic-ratelimit-output-tokens-limit:
70 | - '80000'
71 | anthropic-ratelimit-output-tokens-remaining:
72 | - '80000'
73 | anthropic-ratelimit-output-tokens-reset:
74 | - '2024-12-05T22:04:13Z'
75 | anthropic-ratelimit-requests-limit:
76 | - '4000'
77 | anthropic-ratelimit-requests-remaining:
78 | - '3999'
79 | anthropic-ratelimit-requests-reset:
80 | - '2024-12-05T22:04:13Z'
81 | anthropic-ratelimit-tokens-limit:
82 | - '480000'
83 | anthropic-ratelimit-tokens-remaining:
84 | - '480000'
85 | anthropic-ratelimit-tokens-reset:
86 | - '2024-12-05T22:04:13Z'
87 | content-length:
88 | - '296'
89 | request-id:
90 | - req_01TvAUtsH6Hdsj1YJrdMKmov
91 | via:
92 | - 1.1 google
93 | status:
94 | code: 200
95 | message: OK
96 | version: 1
97 |
--------------------------------------------------------------------------------
/tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_async[async-openai].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model":
4 | "gpt-4o-mini"}'
5 | headers:
6 | accept:
7 | - application/json
8 | accept-encoding:
9 | - gzip, deflate
10 | connection:
11 | - keep-alive
12 | content-length:
13 | - '90'
14 | content-type:
15 | - application/json
16 | host:
17 | - api.openai.com
18 | user-agent:
19 | - AsyncOpenAI/Python 1.55.0
20 | x-stainless-arch:
21 | - arm64
22 | x-stainless-async:
23 | - async:asyncio
24 | x-stainless-lang:
25 | - python
26 | x-stainless-os:
27 | - MacOS
28 | x-stainless-package-version:
29 | - 1.55.0
30 | x-stainless-retry-count:
31 | - '0'
32 | x-stainless-runtime:
33 | - CPython
34 | x-stainless-runtime-version:
35 | - 3.11.10
36 | method: POST
37 | uri: https://api.openai.com/v1/chat/completions
38 | response:
39 | body:
40 | string: "{\n \"id\": \"chatcmpl-AbEFAncbnpE6Od0bEuqJ2jnc1IqLo\",\n \"object\":
41 | \"chat.completion\",\n \"created\": 1733436252,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n
42 | \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\":
43 | \"assistant\",\n \"content\": \"Hello! I'm just a program, but I'm
44 | here and ready to assist you. How can I help you today?\",\n \"refusal\":
45 | null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n
46 | \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 13,\n \"completion_tokens\":
47 | 23,\n \"total_tokens\": 36,\n \"prompt_tokens_details\": {\n \"cached_tokens\":
48 | 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\":
49 | {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\":
50 | 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"system_fingerprint\":
51 | \"fp_0705bf87c0\"\n}\n"
52 | headers:
53 | CF-Cache-Status:
54 | - DYNAMIC
55 | CF-RAY:
56 | - 8ed754a2299831dd-MIA
57 | Connection:
58 | - keep-alive
59 | Content-Type:
60 | - application/json
61 | Date:
62 | - Thu, 05 Dec 2024 22:04:13 GMT
63 | Server:
64 | - cloudflare
65 | Set-Cookie:
66 | - __cf_bm=nHZCNCDMfCHJqFbcd2_XkcnIcFbzHRbCkwTZ3gIwOcY-1733436253-1.0.1.1-RWOKr2iIFIYlKTEypVjPCRHSbSGYb537RmI8WyAkEOH6NndLax0K2FEev9tiJoBT9kBAjamGvQuzvkSA5jdkrw;
67 | path=/; expires=Thu, 05-Dec-24 22:34:13 GMT; domain=.api.openai.com; HttpOnly;
68 | Secure; SameSite=None
69 | - _cfuvid=vsNSK7Vpu5wJTa0Tc99hhlEkhz5sVR4OtPP6jpH4VRk-1733436253076-0.0.1.1-604800000;
70 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
71 | Transfer-Encoding:
72 | - chunked
73 | X-Content-Type-Options:
74 | - nosniff
75 | access-control-expose-headers:
76 | - X-Request-ID
77 | alt-svc:
78 | - h3=":443"; ma=86400
79 | content-length:
80 | - '841'
81 | openai-organization:
82 | - not-diamond-bdz8cg
83 | openai-processing-ms:
84 | - '445'
85 | openai-version:
86 | - '2020-10-01'
87 | strict-transport-security:
88 | - max-age=31536000; includeSubDomains; preload
89 | x-ratelimit-limit-requests:
90 | - '30000'
91 | x-ratelimit-limit-tokens:
92 | - '150000000'
93 | x-ratelimit-remaining-requests:
94 | - '29999'
95 | x-ratelimit-remaining-tokens:
96 | - '149999978'
97 | x-ratelimit-reset-requests:
98 | - 2ms
99 | x-ratelimit-reset-tokens:
100 | - 0s
101 | x-request-id:
102 | - req_22ad18fdbfdbce34e3d5ed53a6ac609e
103 | status:
104 | code: 200
105 | message: OK
106 | version: 1
107 |
--------------------------------------------------------------------------------
/tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_async_create[async-anthropic].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "Hello, how
4 | are you?"}], "model": "claude-3-5-haiku-20241022"}'
5 | headers:
6 | accept:
7 | - application/json
8 | accept-encoding:
9 | - gzip, deflate
10 | anthropic-version:
11 | - '2023-06-01'
12 | connection:
13 | - keep-alive
14 | content-length:
15 | - '124'
16 | content-type:
17 | - application/json
18 | host:
19 | - api.anthropic.com
20 | user-agent:
21 | - AsyncAnthropic/Python 0.39.0
22 | x-api-key:
23 | - MASKED-API-KEY-FOR-TESTING
24 | x-stainless-arch:
25 | - arm64
26 | x-stainless-async:
27 | - async:asyncio
28 | x-stainless-lang:
29 | - python
30 | x-stainless-os:
31 | - MacOS
32 | x-stainless-package-version:
33 | - 0.39.0
34 | x-stainless-retry-count:
35 | - '0'
36 | x-stainless-runtime:
37 | - CPython
38 | x-stainless-runtime-version:
39 | - 3.11.10
40 | method: POST
41 | uri: https://api.anthropic.com/v1/messages
42 | response:
43 | body:
44 | string: '{"id":"msg_01Rewjq1gNNLooVwwqFgrGc8","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''m
45 | doing well, thanks for asking! How are you today? Is there anything I can
46 | help you with?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":13,"output_tokens":26}}'
47 | headers:
48 | CF-Cache-Status:
49 | - DYNAMIC
50 | CF-RAY:
51 | - 8edf71fda9a5da9f-MIA
52 | Connection:
53 | - keep-alive
54 | Content-Type:
55 | - application/json
56 | Date:
57 | - Fri, 06 Dec 2024 21:42:21 GMT
58 | Server:
59 | - cloudflare
60 | Transfer-Encoding:
61 | - chunked
62 | X-Robots-Tag:
63 | - none
64 | anthropic-ratelimit-input-tokens-limit:
65 | - '400000'
66 | anthropic-ratelimit-input-tokens-remaining:
67 | - '400000'
68 | anthropic-ratelimit-input-tokens-reset:
69 | - '2024-12-06T21:42:21Z'
70 | anthropic-ratelimit-output-tokens-limit:
71 | - '80000'
72 | anthropic-ratelimit-output-tokens-remaining:
73 | - '80000'
74 | anthropic-ratelimit-output-tokens-reset:
75 | - '2024-12-06T21:42:21Z'
76 | anthropic-ratelimit-requests-limit:
77 | - '4000'
78 | anthropic-ratelimit-requests-remaining:
79 | - '3999'
80 | anthropic-ratelimit-requests-reset:
81 | - '2024-12-06T21:42:21Z'
82 | anthropic-ratelimit-tokens-limit:
83 | - '480000'
84 | anthropic-ratelimit-tokens-remaining:
85 | - '480000'
86 | anthropic-ratelimit-tokens-reset:
87 | - '2024-12-06T21:42:21Z'
88 | content-length:
89 | - '332'
90 | request-id:
91 | - req_01EeR56CGL4p7RCzSgbZiucu
92 | via:
93 | - 1.1 google
94 | status:
95 | code: 200
96 | message: OK
97 | version: 1
98 |
--------------------------------------------------------------------------------
/tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_async_create[async-azure-openai].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model":
4 | "gpt-4o-mini"}'
5 | headers:
6 | accept:
7 | - application/json
8 | accept-encoding:
9 | - gzip, deflate, zstd
10 | api-key:
11 | - VkhQnoqnZjxaq2NSprIB6ElTip9DgcBAYYA9s8DieI0U2q8C61C0JQQJ99BBACYeBjFXJ3w3AAABACOGP4m6
12 | connection:
13 | - keep-alive
14 | content-length:
15 | - '90'
16 | content-type:
17 | - application/json
18 | host:
19 | - notdiamond-azure-openai.openai.azure.com
20 | user-agent:
21 | - AsyncAzureOpenAI/Python 1.68.2
22 | x-stainless-arch:
23 | - x64
24 | x-stainless-async:
25 | - async:asyncio
26 | x-stainless-lang:
27 | - python
28 | x-stainless-os:
29 | - Linux
30 | x-stainless-package-version:
31 | - 1.68.2
32 | x-stainless-read-timeout:
33 | - '20.0'
34 | x-stainless-retry-count:
35 | - '0'
36 | x-stainless-runtime:
37 | - CPython
38 | x-stainless-runtime-version:
39 | - 3.12.3
40 | method: POST
41 | uri: https://notdiamond-azure-openai.openai.azure.com/openai/deployments/gpt-4o-mini/chat/completions?api-version=2024-07-01-preview
42 | response:
43 | body:
44 | string: '{"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"Hello!
45 | I''m just a program, but I''m here and ready to help you. How can I assist
46 | you today?","refusal":null,"role":"assistant"}}],"created":1742891954,"id":"chatcmpl-BEu6UuKjF02da3tOs1IUoI1rzmNxm","model":"gpt-4o-mini-2024-07-18","object":"chat.completion","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}],"system_fingerprint":"fp_b705f0c291","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":13,"prompt_tokens_details":{"audio_tokens":0,"cached_tokens":0},"total_tokens":37}}
47 |
48 | '
49 | headers:
50 | Content-Length:
51 | - '1112'
52 | Content-Type:
53 | - application/json
54 | Date:
55 | - Tue, 25 Mar 2025 08:39:14 GMT
56 | Strict-Transport-Security:
57 | - max-age=31536000; includeSubDomains; preload
58 | apim-request-id:
59 | - be2f6346-6c44-4a1a-ac24-23f8e17b4c37
60 | azureml-model-session:
61 | - v20250319-1-164616836
62 | cmp-upstream-response-duration:
63 | - '587'
64 | ms-azureml-model-time:
65 | - '635'
66 | x-accel-buffering:
67 | - 'no'
68 | x-aml-cluster:
69 | - hyena-westus3-01
70 | x-content-type-options:
71 | - nosniff
72 | x-envoy-upstream-service-time:
73 | - '638'
74 | x-ms-client-request-id:
75 | - be2f6346-6c44-4a1a-ac24-23f8e17b4c37
76 | x-ms-rai-invoked:
77 | - 'true'
78 | x-ms-region:
79 | - East US
80 | x-ratelimit-limit-requests:
81 | - '20000'
82 | x-ratelimit-limit-tokens:
83 | - '2000000'
84 | x-ratelimit-remaining-requests:
85 | - '19996'
86 | x-ratelimit-remaining-tokens:
87 | - '1995630'
88 | x-request-id:
89 | - 21402e2a-3ef5-421e-a4f2-b6f1acbab9d0
90 | status:
91 | code: 200
92 | message: OK
93 | version: 1
94 |
--------------------------------------------------------------------------------
/tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_create[anthropic].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "Hello, how
4 | are you?"}], "model": "claude-3-5-haiku-20241022"}'
5 | headers:
6 | accept:
7 | - application/json
8 | accept-encoding:
9 | - gzip, deflate
10 | anthropic-version:
11 | - '2023-06-01'
12 | connection:
13 | - keep-alive
14 | content-length:
15 | - '124'
16 | content-type:
17 | - application/json
18 | host:
19 | - api.anthropic.com
20 | user-agent:
21 | - Anthropic/Python 0.39.0
22 | x-api-key:
23 | - MASKED-API-KEY-FOR-TESTING
24 | x-stainless-arch:
25 | - arm64
26 | x-stainless-async:
27 | - 'false'
28 | x-stainless-lang:
29 | - python
30 | x-stainless-os:
31 | - MacOS
32 | x-stainless-package-version:
33 | - 0.39.0
34 | x-stainless-retry-count:
35 | - '0'
36 | x-stainless-runtime:
37 | - CPython
38 | x-stainless-runtime-version:
39 | - 3.11.10
40 | method: POST
41 | uri: https://api.anthropic.com/v1/messages
42 | response:
43 | body:
44 | string: '{"id":"msg_01FZDC9VQuPxgjxWaurydxLV","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''m
45 | doing well, thank you for asking! How are you today? Is there anything I can
46 | help you with?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":13,"output_tokens":27}}'
47 | headers:
48 | CF-Cache-Status:
49 | - DYNAMIC
50 | CF-RAY:
51 | - 8edf71f64d1a09ce-MIA
52 | Connection:
53 | - keep-alive
54 | Content-Type:
55 | - application/json
56 | Date:
57 | - Fri, 06 Dec 2024 21:42:20 GMT
58 | Server:
59 | - cloudflare
60 | Transfer-Encoding:
61 | - chunked
62 | X-Robots-Tag:
63 | - none
64 | anthropic-ratelimit-input-tokens-limit:
65 | - '400000'
66 | anthropic-ratelimit-input-tokens-remaining:
67 | - '400000'
68 | anthropic-ratelimit-input-tokens-reset:
69 | - '2024-12-06T21:42:20Z'
70 | anthropic-ratelimit-output-tokens-limit:
71 | - '80000'
72 | anthropic-ratelimit-output-tokens-remaining:
73 | - '80000'
74 | anthropic-ratelimit-output-tokens-reset:
75 | - '2024-12-06T21:42:20Z'
76 | anthropic-ratelimit-requests-limit:
77 | - '4000'
78 | anthropic-ratelimit-requests-remaining:
79 | - '3999'
80 | anthropic-ratelimit-requests-reset:
81 | - '2024-12-06T21:42:19Z'
82 | anthropic-ratelimit-tokens-limit:
83 | - '480000'
84 | anthropic-ratelimit-tokens-remaining:
85 | - '480000'
86 | anthropic-ratelimit-tokens-reset:
87 | - '2024-12-06T21:42:20Z'
88 | content-length:
89 | - '335'
90 | request-id:
91 | - req_01TAPLdMRJWY6m8vJGbZCmCQ
92 | via:
93 | - 1.1 google
94 | status:
95 | code: 200
96 | message: OK
97 | version: 1
98 |
--------------------------------------------------------------------------------
/tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_create[azure-openai].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model":
4 | "gpt-4o-mini"}'
5 | headers:
6 | accept:
7 | - application/json
8 | accept-encoding:
9 | - gzip, deflate, zstd
10 | api-key:
11 | - VkhQnoqnZjxaq2NSprIB6ElTip9DgcBAYYA9s8DieI0U2q8C61C0JQQJ99BBACYeBjFXJ3w3AAABACOGP4m6
12 | connection:
13 | - keep-alive
14 | content-length:
15 | - '90'
16 | content-type:
17 | - application/json
18 | host:
19 | - notdiamond-azure-openai.openai.azure.com
20 | user-agent:
21 | - AzureOpenAI/Python 1.68.2
22 | x-stainless-arch:
23 | - x64
24 | x-stainless-async:
25 | - 'false'
26 | x-stainless-lang:
27 | - python
28 | x-stainless-os:
29 | - Linux
30 | x-stainless-package-version:
31 | - 1.68.2
32 | x-stainless-read-timeout:
33 | - '20.0'
34 | x-stainless-retry-count:
35 | - '0'
36 | x-stainless-runtime:
37 | - CPython
38 | x-stainless-runtime-version:
39 | - 3.12.3
40 | method: POST
41 | uri: https://notdiamond-azure-openai.openai.azure.com/openai/deployments/gpt-4o-mini/chat/completions?api-version=2024-07-01-preview
42 | response:
43 | body:
44 | string: '{"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"Hello!
45 | I''m just a computer program, so I don''t have feelings, but I''m here and
46 | ready to help you. How can I assist you today?","refusal":null,"role":"assistant"}}],"created":1742891952,"id":"chatcmpl-BEu6SSU7lz2f9yspHEGqKXFVAbiMP","model":"gpt-4o-mini-2024-07-18","object":"chat.completion","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}],"system_fingerprint":"fp_ded0d14823","usage":{"completion_tokens":31,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":13,"prompt_tokens_details":{"audio_tokens":0,"cached_tokens":0},"total_tokens":44}}
47 |
48 | '
49 | headers:
50 | Content-Length:
51 | - '1147'
52 | Content-Type:
53 | - application/json
54 | Date:
55 | - Tue, 25 Mar 2025 08:39:12 GMT
56 | Strict-Transport-Security:
57 | - max-age=31536000; includeSubDomains; preload
58 | apim-request-id:
59 | - af3a9be3-f831-4142-b782-d63640aa0369
60 | azureml-model-session:
61 | - v20250319-1-164616836
62 | cmp-upstream-response-duration:
63 | - '512'
64 | ms-azureml-model-time:
65 | - '549'
66 | x-accel-buffering:
67 | - 'no'
68 | x-aml-cluster:
69 | - hyena-westus3-02
70 | x-content-type-options:
71 | - nosniff
72 | x-envoy-upstream-service-time:
73 | - '551'
74 | x-ms-client-request-id:
75 | - af3a9be3-f831-4142-b782-d63640aa0369
76 | x-ms-rai-invoked:
77 | - 'true'
78 | x-ms-region:
79 | - East US
80 | x-ratelimit-limit-requests:
81 | - '20000'
82 | x-ratelimit-limit-tokens:
83 | - '2000000'
84 | x-ratelimit-remaining-requests:
85 | - '19997'
86 | x-ratelimit-remaining-tokens:
87 | - '1996275'
88 | x-request-id:
89 | - 2fecd1e8-fcfa-439a-8754-b8caff264432
90 | status:
91 | code: 200
92 | message: OK
93 | version: 1
94 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[anthropic-claude-3-5-sonnet-20240620-langchain_anthropic.ChatAnthropic].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "anthropic", "model": "claude-3-5-sonnet-20240620", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '314'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=d1ba7bb2-1139-49a0, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13df5fa78eec-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:55 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - d1ba7bb2-1139-49a0
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[anthropic-claude-3-haiku-20240307-langchain_anthropic.ChatAnthropic].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "anthropic", "model": "claude-3-haiku-20240307", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '311'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=32e65e02-7163-476f, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13ebee865afa-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:57 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 32e65e02-7163-476f
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[anthropic-claude-3-opus-20240229-langchain_anthropic.ChatAnthropic].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "anthropic", "model": "claude-3-opus-20240229", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '310'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=8178b5b5-5d5d-4a3e, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13e46dd1c314-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:56 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 8178b5b5-5d5d-4a3e
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[anthropic-claude-3-sonnet-20240229-langchain_anthropic.ChatAnthropic].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "anthropic", "model": "claude-3-sonnet-20240229", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '312'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=18e74451-fdd1-4daa, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13e87909dfcc-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:57 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 18e74451-fdd1-4daa
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[cohere-command-r-langchain_cohere.ChatCohere].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "cohere", "model": "command-r", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '294'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=e8b23219-8ce1-474b, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d1435df6a5b78-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:09 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - e8b23219-8ce1-474b
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[cohere-command-r-plus-langchain_cohere.ChatCohere].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "cohere", "model": "command-r-plus", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '299'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=5ddfef53-5588-4818, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d14323a805b6c-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:08 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 5ddfef53-5588-4818
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[google-gemini-1.5-flash-latest-langchain_google_genai.ChatGoogleGenerativeAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "google", "model": "gemini-1.5-flash-latest", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '308'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=a64972ee-d870-493e, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13f6ea343250-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:59 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - a64972ee-d870-493e
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[google-gemini-1.5-pro-latest-langchain_google_genai.ChatGoogleGenerativeAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "google", "model": "gemini-1.5-pro-latest", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '306'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=30b056f5-1ab7-46ee, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13f17a973dc0-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:58 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 30b056f5-1ab7-46ee
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-codestral-latest-langchain_mistralai.ChatMistralAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "mistral", "model": "codestral-latest", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '302'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=1728d3df-69c9-4524, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13fc6e665a5f-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:00 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 1728d3df-69c9-4524
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-mistral-large-2402-langchain_mistralai.ChatMistralAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "mistral", "model": "mistral-large-2402", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '304'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=f3150336-ac6a-4f64, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d14056ef15bad-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:01 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - f3150336-ac6a-4f64
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-mistral-large-2407-langchain_mistralai.ChatMistralAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "mistral", "model": "mistral-large-2407", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '304'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=d5603514-a7fa-4a40, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d14029b91d0cb-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:01 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - d5603514-a7fa-4a40
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-mistral-medium-latest-langchain_mistralai.ChatMistralAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "mistral", "model": "mistral-medium-latest", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '307'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=953f08fc-c5c1-495e, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d1408c9379aa0-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:02 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 953f08fc-c5c1-495e
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-mistral-small-latest-langchain_mistralai.ChatMistralAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "mistral", "model": "mistral-small-latest", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '306'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=7aebc438-5a94-45b1, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d140b5d425ba3-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:02 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 7aebc438-5a94-45b1
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-open-mistral-7b-langchain_mistralai.ChatMistralAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "mistral", "model": "open-mistral-7b", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '301'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=cb026621-bf3c-487f, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d14107888d0ea-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:03 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - cb026621-bf3c-487f
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-open-mixtral-8x22b-langchain_mistralai.ChatMistralAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "mistral", "model": "open-mixtral-8x22b", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '304'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=1ce7a93b-eafe-46a8, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13f9aa335a9b-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:59 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 1ce7a93b-eafe-46a8
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-open-mixtral-8x7b-langchain_mistralai.ChatMistralAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "mistral", "model": "open-mixtral-8x7b", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '303'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=d4d6a167-30ab-4d2d, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13ffedafd0c4-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:00 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - d4d6a167-30ab-4d2d
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-3.5-turbo-0125-langchain_openai.ChatOpenAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "openai", "model": "gpt-3.5-turbo-0125", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '303'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=43551482-b870-4a26, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13d8bd658ef2-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:54 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 43551482-b870-4a26
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4-0125-preview-langchain_openai.ChatOpenAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "openai", "model": "gpt-4-0125-preview", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '303'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=deb88868-fc19-4780, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13cf4d41c26a-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:53 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - deb88868-fc19-4780
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4-0613-langchain_openai.ChatOpenAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "openai", "model": "gpt-4-0613", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '295'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=fe14af84-ff88-40c0, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13d58f97d0d7-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:54 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - fe14af84-ff88-40c0
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4-1106-preview-langchain_openai.ChatOpenAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "openai", "model": "gpt-4-1106-preview", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '303'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=7c0b1238-78f1-4d4c, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13d29b9e6e9e-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:53 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 7c0b1238-78f1-4d4c
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4-turbo-2024-04-09-langchain_openai.ChatOpenAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "openai", "model": "gpt-4-turbo-2024-04-09", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '307'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=47751cf0-39d4-466e, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13cc7d7589c3-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:52 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 47751cf0-39d4-466e
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4o-2024-05-13-langchain_openai.ChatOpenAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "openai", "model": "gpt-4o-2024-05-13", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '302'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=7089b607-7066-4c50, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13c92dedc2fa-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:52 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 7089b607-7066-4c50
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4o-2024-08-06-langchain_openai.ChatOpenAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "openai", "model": "gpt-4o-2024-08-06", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '302'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=6f3319f5-6e71-4aa4, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13c4f9d8c27d-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:51 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 6f3319f5-6e71-4aa4
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4o-langchain_openai.ChatOpenAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "openai", "model": "gpt-4o", "is_custom": false, "context_length":
5 | null, "input_price": null, "output_price": null, "latency": null}], "metric":
6 | "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '291'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=54dc61bb-1e8f-4a07, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13c25b982693-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:51 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 54dc61bb-1e8f-4a07
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4o-mini-2024-07-18-langchain_openai.ChatOpenAI].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "openai", "model": "gpt-4o-mini-2024-07-18", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '307'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=2e0bd7b2-8cc2-4377, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d13db6bb65bb2-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:35:55 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 2e0bd7b2-8cc2-4377
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Llama-3-70b-chat-hf-langchain_together.ChatTogether].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "togetherai", "model": "Llama-3-70b-chat-hf", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '308'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=dcb4a717-b0d0-4453, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d1412f91b1c23-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:03 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - dcb4a717-b0d0-4453
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Llama-3-8b-chat-hf-langchain_together.ChatTogether].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "togetherai", "model": "Llama-3-8b-chat-hf", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '307'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=fb0d20ad-5e6e-49fd, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d1415cdb125c0-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:04 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - fb0d20ad-5e6e-49fd
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Meta-Llama-3.1-405B-Instruct-Turbo-langchain_together.ChatTogether].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "togetherai", "model": "Meta-Llama-3.1-405B-Instruct-Turbo", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '323'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=4775345a-1f88-43b7, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d141f198e1c9c-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:05 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 4775345a-1f88-43b7
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Meta-Llama-3.1-70B-Instruct-Turbo-langchain_together.ChatTogether].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "togetherai", "model": "Meta-Llama-3.1-70B-Instruct-Turbo", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '322'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=a13267ad-cf8a-4f48, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d141ba8432bd8-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:05 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - a13267ad-cf8a-4f48
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Meta-Llama-3.1-8B-Instruct-Turbo-langchain_together.ChatTogether].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "togetherai", "model": "Meta-Llama-3.1-8B-Instruct-Turbo", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '321'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=f7821e12-47f1-4480, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d14193a602bd8-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:04 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - f7821e12-47f1-4480
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Mistral-7B-Instruct-v0.2-langchain_together.ChatTogether].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "togetherai", "model": "Mistral-7B-Instruct-v0.2", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '313'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=08eaf899-2575-47a7, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d142eb8055db9-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:08 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 08eaf899-2575-47a7
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Mixtral-8x22B-Instruct-v0.1-langchain_together.ChatTogether].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "togetherai", "model": "Mixtral-8x22B-Instruct-v0.1", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '316'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=b6a7ffcb-f95e-4c7c, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d14261cb8c297-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:06 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - b6a7ffcb-f95e-4c7c
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Mixtral-8x7B-Instruct-v0.1-langchain_together.ChatTogether].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "togetherai", "model": "Mixtral-8x7B-Instruct-v0.1", "is_custom":
5 | false, "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '315'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=606ee8fb-d82b-472b, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d142c09c41ca0-SOF
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:07 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 606ee8fb-d82b-472b
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Qwen2-72B-Instruct-langchain_together.ChatTogether].yaml:
--------------------------------------------------------------------------------
1 | interactions:
2 | - request:
3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers":
4 | [{"provider": "togetherai", "model": "Qwen2-72B-Instruct", "is_custom": false,
5 | "context_length": null, "input_price": null, "output_price": null, "latency":
6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}'
7 | headers:
8 | Accept:
9 | - '*/*'
10 | Accept-Encoding:
11 | - gzip, deflate, zstd
12 | Connection:
13 | - keep-alive
14 | Content-Length:
15 | - '307'
16 | User-Agent:
17 | - langchain-community/0.3.41
18 | content-type:
19 | - application/json
20 | method: POST
21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect
22 | response:
23 | body:
24 | string: '{"detail":"requestID=793a0740-e1b6-44f3, Could not find valid API key"}'
25 | headers:
26 | CF-RAY:
27 | - 925d1421cb9b9796-VIE
28 | Connection:
29 | - keep-alive
30 | Content-Type:
31 | - application/json
32 | Date:
33 | - Tue, 25 Mar 2025 08:36:06 GMT
34 | Server:
35 | - cloudflare
36 | Transfer-Encoding:
37 | - chunked
38 | alt-svc:
39 | - h3=":443"; ma=86400
40 | cf-cache-status:
41 | - DYNAMIC
42 | rndr-id:
43 | - 793a0740-e1b6-44f3
44 | vary:
45 | - Accept-Encoding
46 | x-render-origin-server:
47 | - uvicorn
48 | status:
49 | code: 401
50 | message: Unauthorized
51 | version: 1
52 |
--------------------------------------------------------------------------------
/tests/test_toolkit/rag/conftest.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import pytest
3 | from llama_index.core import SimpleDirectoryReader
4 |
5 | from notdiamond.llms.config import EmbeddingConfig, LLMConfig
6 | from notdiamond.toolkit.rag.evaluation_dataset import (
7 | RAGEvaluationDataset,
8 | RAGSample,
9 | )
10 |
11 |
12 | def format_prompt(user_input: str, retrieved_contexts: list[str]) -> str:
13 | context = "\n".join(retrieved_contexts)
14 | prompt = f"""
15 | Use the following context to answer the question.
16 |
17 | Context: {context}
18 |
19 | Question: {user_input}
20 | """
21 | return prompt
22 |
23 |
24 | @pytest.fixture
25 | def user_input():
26 | return "What's the capital of France?"
27 |
28 |
29 | @pytest.fixture
30 | def retrieved_contexts():
31 | return ["Paris is the capital and most populous city of France."]
32 |
33 |
34 | @pytest.fixture
35 | def response():
36 | return "The capital of France is Paris."
37 |
38 |
39 | @pytest.fixture
40 | def reference():
41 | return "Paris"
42 |
43 |
44 | @pytest.fixture
45 | def gpt_4o():
46 | return LLMConfig.from_string("openai/gpt-4o")
47 |
48 |
49 | @pytest.fixture
50 | def sonnet_3_5():
51 | return LLMConfig.from_string("anthropic/claude-3-5-sonnet-20241022")
52 |
53 |
54 | @pytest.fixture
55 | def openai_embedding():
56 | return EmbeddingConfig.from_string("openai/text-embedding-3-large")
57 |
58 |
59 | @pytest.fixture
60 | def dataset(gpt_4o, user_input, retrieved_contexts, response, reference):
61 | samples = [
62 | RAGSample(
63 | user_input=user_input,
64 | retrieved_contexts=retrieved_contexts,
65 | response=response,
66 | reference=reference,
67 | generation_prompt=format_prompt(user_input, retrieved_contexts),
68 | generator_llm=str(gpt_4o),
69 | ),
70 | RAGSample(
71 | user_input=user_input,
72 | retrieved_contexts=retrieved_contexts,
73 | response=response,
74 | reference=reference,
75 | generation_prompt=format_prompt(user_input, retrieved_contexts),
76 | generator_llm=str(gpt_4o),
77 | ),
78 | ]
79 | return RAGEvaluationDataset(samples)
80 |
81 |
82 | @pytest.fixture
83 | def pandas_dataset(
84 | gpt_4o, user_input, retrieved_contexts, response, reference
85 | ):
86 | data = {
87 | "user_input": [user_input, user_input],
88 | "retrieved_contexts": [retrieved_contexts, retrieved_contexts],
89 | "response": [response, response],
90 | "reference": [reference, reference],
91 | "generation_prompt": [
92 | format_prompt(user_input, retrieved_contexts),
93 | format_prompt(user_input, retrieved_contexts),
94 | ],
95 | "generator_llm": [str(gpt_4o), str(gpt_4o)],
96 | }
97 | df = pd.DataFrame(data=data)
98 | return df
99 |
100 |
101 | @pytest.fixture
102 | def llamaindex_documents():
103 | loader = SimpleDirectoryReader(
104 | input_files=[
105 | "tests/static/airbnb_tos.md",
106 | ]
107 | )
108 | docs = loader.load_data()
109 | return docs
110 |
111 |
112 | @pytest.fixture
113 | def test_queries():
114 | return ["summarize airbnb's ToS", "What are the cancellation policies?"]
115 |
--------------------------------------------------------------------------------
/tests/test_toolkit/rag/test_data_gen.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import pytest
3 | from llama_index.core import SimpleDirectoryReader
4 | from ragas.embeddings import BaseRagasEmbeddings
5 | from ragas.llms import BaseRagasLLM
6 |
7 | from notdiamond.llms.config import EmbeddingConfig, LLMConfig
8 | from notdiamond.toolkit.rag.document_loaders import DirectoryLoader
9 | from notdiamond.toolkit.rag.llms import get_embedding, get_llm
10 | from notdiamond.toolkit.rag.testset import TestDataGenerator
11 |
12 |
13 | @pytest.fixture
14 | def test_data_langchain_docs():
15 | loader = DirectoryLoader("tests/static/", glob="airbnb_tos.md")
16 | docs = loader.load()
17 | return docs
18 |
19 |
20 | @pytest.fixture
21 | def test_data_llamaindex_docs():
22 | loader = SimpleDirectoryReader(
23 | input_files=[
24 | "tests/static/airbnb_tos.md",
25 | ]
26 | )
27 | docs = loader.load_data()
28 | return docs
29 |
30 |
31 | @pytest.fixture
32 | def generator_llm():
33 | return get_llm("openai/gpt-4o-mini")
34 |
35 |
36 | @pytest.fixture
37 | def generator_embedding():
38 | return get_embedding("openai/text-embedding-3-small")
39 |
40 |
41 | def test_get_llm_by_config():
42 | llm_config = LLMConfig(provider="openai", model="gpt-4o-mini")
43 | llm = get_llm(llm_config)
44 | assert isinstance(llm, BaseRagasLLM)
45 |
46 |
47 | def test_get_embedding_by_config():
48 | embed_config = EmbeddingConfig(
49 | provider="openai", model="text-embedding-3-small"
50 | )
51 | embed = get_embedding(embed_config)
52 | assert isinstance(embed, BaseRagasEmbeddings)
53 |
54 |
55 | def test_dataset_generator_langchain_docs(
56 | test_data_langchain_docs, generator_llm, generator_embedding
57 | ):
58 | generator = TestDataGenerator(
59 | llm=generator_llm, embedding_model=generator_embedding
60 | )
61 | dataset = generator.generate_from_docs(
62 | test_data_langchain_docs, testset_size=2
63 | )
64 | assert len(dataset) == 2
65 | assert isinstance(dataset, pd.DataFrame)
66 |
67 |
68 | def test_dataset_generator_llamaindex_docs(
69 | test_data_llamaindex_docs, generator_llm, generator_embedding
70 | ):
71 | generator = TestDataGenerator(
72 | llm=generator_llm, embedding_model=generator_embedding
73 | )
74 | dataset = generator.generate_from_docs(
75 | test_data_llamaindex_docs, testset_size=2
76 | )
77 | assert len(dataset) == 2
78 | assert isinstance(dataset, pd.DataFrame)
79 |
--------------------------------------------------------------------------------
/tests/test_toolkit/rag/test_evaluation.py:
--------------------------------------------------------------------------------
1 | import pytest
2 |
3 | from notdiamond.toolkit.rag.evaluation import evaluate
4 | from notdiamond.toolkit.rag.evaluation_dataset import (
5 | RAGEvaluationDataset,
6 | RAGSample,
7 | )
8 | from notdiamond.toolkit.rag.llms import get_embedding, get_llm
9 | from notdiamond.toolkit.rag.metrics import Faithfulness, SemanticSimilarity
10 |
11 |
12 | def test_dataset_from_pandas(pandas_dataset):
13 | dataset = RAGEvaluationDataset.from_pandas(pandas_dataset)
14 | assert isinstance(dataset, RAGEvaluationDataset)
15 |
16 |
17 | def test_dataset_format(dataset):
18 | assert len(dataset) == 2
19 | assert isinstance(dataset[0], RAGSample)
20 | assert isinstance(dataset[1], RAGSample)
21 |
22 |
23 | @pytest.mark.vcr
24 | def test_evaluate(dataset, gpt_4o, sonnet_3_5, openai_embedding):
25 | evaluator_llm = get_llm(gpt_4o)
26 | evaluator_embedding = get_embedding(openai_embedding)
27 |
28 | metrics = [
29 | Faithfulness(llm=evaluator_llm),
30 | SemanticSimilarity(embeddings=evaluator_embedding),
31 | ]
32 |
33 | generator_llms = [gpt_4o, sonnet_3_5]
34 |
35 | results = evaluate(
36 | dataset=dataset,
37 | metrics=metrics,
38 | generator_llms=generator_llms,
39 | )
40 |
41 | expected_results_keys = [str(gpt_4o), str(sonnet_3_5)]
42 | assert all([key in expected_results_keys for key in results.keys()])
43 |
44 | gpt_4o_result = results[str(gpt_4o)]
45 | expected_results_columns = [
46 | "user_input",
47 | "generation_prompt",
48 | "retrieved_contexts",
49 | "response",
50 | "reference",
51 | "faithfulness",
52 | "semantic_similarity",
53 | ]
54 | assert all(
55 | [
56 | col in expected_results_columns
57 | for col in list(gpt_4o_result.columns)
58 | ]
59 | )
60 |
--------------------------------------------------------------------------------
/tests/test_toolkit/rag/test_workflow.py:
--------------------------------------------------------------------------------
1 | from typing import Annotated, Any, List
2 |
3 | import pytest
4 |
5 | from notdiamond.toolkit.rag.workflow import BaseNDRagWorkflow, IntValueRange
6 |
7 |
8 | class TestNDRagWorkflow(BaseNDRagWorkflow):
9 | parameter_specs = {
10 | "chunk_size": (Annotated[int, IntValueRange(1000, 2500, 500)], 1000)
11 | }
12 |
13 | def rag_workflow(self, documents: Any, test_queries: List[str]):
14 | pass
15 |
16 |
17 | def test_set_param_values(llamaindex_documents, test_queries):
18 | workflow = TestNDRagWorkflow(
19 | llamaindex_documents, test_queries, objective_maximize=True
20 | )
21 | workflow._set_param_values({"chunk_size": 1500})
22 | assert workflow.chunk_size == 1500
23 |
24 | with pytest.raises(ValueError):
25 | workflow._set_param_values({"nonexistent_param": 1500})
26 |
27 | with pytest.raises(ValueError):
28 | workflow._set_param_values({"chunk_size": "not an int"})
29 |
30 | # value too low for range
31 | with pytest.raises(ValueError):
32 | workflow._set_param_values({"chunk_size": 100})
33 |
34 |
35 | def test_set_bad_param_values(test_queries, llamaindex_documents):
36 | class BadNDRagWorkflow(BaseNDRagWorkflow):
37 | parameter_specs = {"chunk_size": (int, 1000)}
38 |
39 | def rag_workflow(self, documents: Any, test_queries: List[str]):
40 | pass
41 |
42 | # should fail bc we need ranges
43 | with pytest.raises(ValueError):
44 | BadNDRagWorkflow(
45 | llamaindex_documents, test_queries, objective_maximize=True
46 | )
47 |
48 | class BadNDRagWorkflow2(BaseNDRagWorkflow):
49 | parameter_specs = {"chunk_size": (int, "foo")}
50 |
51 | # should fail bc we need matching types
52 | with pytest.raises(ValueError):
53 | BadNDRagWorkflow2(
54 | llamaindex_documents, test_queries, objective_maximize=True
55 | )
56 |
--------------------------------------------------------------------------------
/tests/test_types.py:
--------------------------------------------------------------------------------
1 | """Tests for the types defined in the Python library"""
2 |
3 |
4 | import sys
5 |
6 | import pytest
7 |
8 | from notdiamond.exceptions import InvalidApiKey, MissingApiKey
9 | from notdiamond.types import NDApiKeyValidator
10 |
11 | sys.path.append("../")
12 |
13 |
14 | def test_api_key_not_string_fails():
15 | """When the API key is not a string, the validation fails."""
16 | with pytest.raises(InvalidApiKey):
17 | NDApiKeyValidator(api_key=1)
18 |
19 | with pytest.raises(InvalidApiKey):
20 | NDApiKeyValidator(api_key=None)
21 |
22 |
23 | def test_api_key_empty_string_fails():
24 | """When the API key is an empty string, the validation fails."""
25 | with pytest.raises(MissingApiKey):
26 | NDApiKeyValidator(api_key="")
27 |
28 |
29 | def test_api_key_passes():
30 | """When the API key is a valid string and not empty, validation passes successfully."""
31 | notdiamond_api_key = NDApiKeyValidator(api_key="NDAPIKEY")
32 | assert notdiamond_api_key.api_key == "NDAPIKEY"
33 |
--------------------------------------------------------------------------------