├── .github ├── actions │ ├── set-working-directory │ │ └── action.yml │ └── setup-python-env │ │ └── action.yml └── workflows │ ├── all-sdk-tests.yml │ ├── build_docs.yml │ ├── coverage.yml │ ├── release.yml │ └── static_docs.yml ├── .gitignore ├── .pre-commit-config.yaml ├── README.md ├── build └── release.sh ├── dist └── .gitkeep ├── docs ├── Makefile ├── _build │ ├── doctrees │ │ ├── environment.pickle │ │ ├── index.doctree │ │ └── source │ │ │ ├── examples.doctree │ │ │ ├── intro.doctree │ │ │ ├── modules.doctree │ │ │ ├── notdiamond.doctree │ │ │ ├── notdiamond.llms.doctree │ │ │ ├── notdiamond.metrics.doctree │ │ │ ├── notdiamond.prompts.doctree │ │ │ ├── notdiamond.toolkit.doctree │ │ │ ├── notdiamond.toolkit.litellm.doctree │ │ │ └── notdiamond.toolkit.rag.doctree │ └── html │ │ ├── .buildinfo │ │ ├── .doctrees │ │ ├── environment.pickle │ │ ├── index.doctree │ │ └── source │ │ │ ├── intro.doctree │ │ │ ├── notdiamond.doctree │ │ │ ├── notdiamond.llms.doctree │ │ │ ├── notdiamond.metrics.doctree │ │ │ ├── notdiamond.toolkit.doctree │ │ │ └── notdiamond.toolkit.rag.doctree │ │ ├── .nojekyll │ │ ├── _modules │ │ ├── index.html │ │ ├── notdiamond │ │ │ ├── _init.html │ │ │ ├── callbacks.html │ │ │ ├── exceptions.html │ │ │ ├── llms │ │ │ │ ├── client.html │ │ │ │ ├── config.html │ │ │ │ ├── llm.html │ │ │ │ ├── provider.html │ │ │ │ ├── providers.html │ │ │ │ └── request.html │ │ │ ├── metrics │ │ │ │ ├── metric.html │ │ │ │ └── request.html │ │ │ ├── prompts.html │ │ │ ├── prompts │ │ │ │ ├── hash.html │ │ │ │ └── prompt.html │ │ │ ├── toolkit │ │ │ │ ├── custom_router.html │ │ │ │ ├── langchain.html │ │ │ │ ├── openai.html │ │ │ │ └── rag │ │ │ │ │ ├── evaluation.html │ │ │ │ │ ├── evaluation_dataset.html │ │ │ │ │ ├── llms.html │ │ │ │ │ ├── testset.html │ │ │ │ │ └── workflow.html │ │ │ └── types.html │ │ ├── pydantic │ │ │ ├── _internal │ │ │ │ └── _repr.html │ │ │ └── main.html │ │ ├── ragas │ │ │ └── dataset_schema.html │ │ └── typing.html │ │ ├── _static │ │ ├── _sphinx_javascript_frameworks_compat.js │ │ ├── basic.css │ │ ├── css │ │ │ ├── badge_only.css │ │ │ ├── fonts │ │ │ │ ├── Roboto-Slab-Bold.woff │ │ │ │ ├── Roboto-Slab-Bold.woff2 │ │ │ │ ├── Roboto-Slab-Regular.woff │ │ │ │ ├── Roboto-Slab-Regular.woff2 │ │ │ │ ├── fontawesome-webfont.eot │ │ │ │ ├── fontawesome-webfont.svg │ │ │ │ ├── fontawesome-webfont.ttf │ │ │ │ ├── fontawesome-webfont.woff │ │ │ │ ├── fontawesome-webfont.woff2 │ │ │ │ ├── lato-bold-italic.woff │ │ │ │ ├── lato-bold-italic.woff2 │ │ │ │ ├── lato-bold.woff │ │ │ │ ├── lato-bold.woff2 │ │ │ │ ├── lato-normal-italic.woff │ │ │ │ ├── lato-normal-italic.woff2 │ │ │ │ ├── lato-normal.woff │ │ │ │ └── lato-normal.woff2 │ │ │ ├── theme.css │ │ │ └── vendor │ │ │ │ └── bootstrap.min.css │ │ ├── doctools.js │ │ ├── documentation_options.js │ │ ├── file.png │ │ ├── jquery.js │ │ ├── js │ │ │ ├── badge_only.js │ │ │ ├── html5shiv-printshiv.min.js │ │ │ ├── html5shiv.min.js │ │ │ ├── searchtools.js │ │ │ ├── theme.js │ │ │ └── vendor │ │ │ │ └── bootstrap.min.js │ │ ├── language_data.js │ │ ├── minus.png │ │ ├── plus.png │ │ ├── pygments.css │ │ ├── searchtools.js │ │ └── sphinx_highlight.js │ │ ├── index.html │ │ ├── objects.inv │ │ ├── py-modindex.html │ │ ├── search.html │ │ ├── searchindex.js │ │ └── source │ │ ├── examples.html │ │ ├── intro.html │ │ ├── modules.html │ │ ├── notdiamond.html │ │ ├── notdiamond.llms.html │ │ ├── notdiamond.metrics.html │ │ ├── notdiamond.prompts.html │ │ ├── notdiamond.toolkit.html │ │ ├── notdiamond.toolkit.litellm.html │ │ └── notdiamond.toolkit.rag.html ├── conf.py ├── index.rst ├── make.bat └── source │ ├── intro.rst │ ├── notdiamond.llms.rst │ ├── notdiamond.metrics.rst │ ├── notdiamond.rst │ ├── notdiamond.toolkit.rag.rst │ └── notdiamond.toolkit.rst ├── notdiamond ├── __init__.py ├── _init.py ├── _utils.py ├── callbacks.py ├── exceptions.py ├── llms │ ├── __init__.py │ ├── client.py │ ├── config.py │ ├── providers.py │ └── request.py ├── metrics │ ├── __init__.py │ ├── metric.py │ └── request.py ├── prompts.py ├── settings.py ├── toolkit │ ├── __init__.py │ ├── _retry.py │ ├── custom_router.py │ ├── langchain.py │ ├── openai.py │ └── rag │ │ ├── __init__.py │ │ ├── document_loaders.py │ │ ├── evaluation.py │ │ ├── evaluation_dataset.py │ │ ├── llms.py │ │ ├── metrics.py │ │ ├── testset.py │ │ └── workflow.py └── types.py ├── poetry.lock ├── pyproject.toml └── tests ├── cassettes └── test_init │ ├── test_async_init_multi_provider_multi_model.yaml │ ├── test_async_init_multi_provider_multi_model_azure_error.yaml │ ├── test_async_init_multi_provider_multi_model_multi_config.yaml │ ├── test_init_multi_provider_multi_model.yaml │ ├── test_init_multi_provider_multi_model_azure_error.yaml │ └── test_init_multi_provider_multi_model_multi_config.yaml ├── conftest.py ├── helpers.py ├── static ├── airbnb_tos.md └── bbh_implicatures.json ├── test_components ├── test_llms │ ├── cassettes │ │ ├── test_callbacks │ │ │ ├── test_latency_tracking_callback.yaml │ │ │ ├── test_llm_start_callback.yaml │ │ │ ├── test_model_select_callback[_NDInvokerClient].yaml │ │ │ └── test_model_select_callback[_NDRouterClient].yaml │ │ ├── test_llm │ │ │ ├── Test_NDLLM.test_async_model_select_with_strings[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_async_model_select_with_strings[_NDClientTarget.ROUTER-_NDRouterClient].yaml │ │ │ ├── Test_NDLLM.test_model_select_with_strings[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_model_select_with_strings[_NDClientTarget.ROUTER-_NDRouterClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_ainvoke_response_model[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_astream_response_model[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_async_tool_calling[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_invoke_response_model[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_invoke_with_curly_braces[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_openai_interface[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_stream_response_model[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_tool_calling[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_tool_calling_astream[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_ndllm_tool_calling_stream[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_preference_id[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_NDLLM.test_preference_id[_NDClientTarget.ROUTER-_NDRouterClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_create_with_default[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_create_with_latency_tracking[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_create_with_max_model_depth[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_create_with_provider_system_prompts[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_llm_with_tradeoff[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_llm_without_ndllm_configs[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_ainvoke[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_astream[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_invoke[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_invoke_claude[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_model_select[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ │ ├── Test_OpenAI_style_input.test_openai_style_input_model_select[_NDClientTarget.ROUTER-_NDRouterClient].yaml │ │ │ └── Test_OpenAI_style_input.test_openai_style_input_stream[_NDClientTarget.INVOKER-_NDInvokerClient].yaml │ │ └── test_llm_request │ │ │ ├── test_async_llm_invoke_with_latency_tracking_success.yaml │ │ │ ├── test_custom_model_attributes.yaml │ │ │ ├── test_llm_invoke_with_latency_tracking_nd_chat_prompt_success.yaml │ │ │ ├── test_llm_invoke_with_latency_tracking_success.yaml │ │ │ └── test_session_linking.yaml │ ├── test_callbacks.py │ ├── test_embedding_config.py │ ├── test_llm.py │ ├── test_llm_request.py │ └── test_provider.py └── test_utils.py ├── test_documentation ├── cassettes │ ├── test_fallback_and_custom │ │ └── test_custom_logic.yaml │ ├── test_function_calling │ │ ├── test_function_calling.yaml │ │ └── test_function_calling_via_rest_api.yaml │ ├── test_getting_started │ │ ├── test_main_example.yaml │ │ ├── test_model_select.yaml │ │ ├── test_pass_array_of_messages.yaml │ │ └── test_programatic_define_llm_configs.yaml │ ├── test_langchain │ │ └── test_streaming.yaml │ ├── test_openrouter │ │ └── test_openrouter_integration.yaml │ ├── test_personalization │ │ └── test_personalization.yaml │ └── test_structured_output │ │ ├── test_structured_output.yaml │ │ └── test_structured_output_streaming.yaml ├── test_fallback_and_custom.py ├── test_function_calling.py ├── test_getting_started.py ├── test_langchain.py ├── test_openrouter.py ├── test_personalization.py └── test_structured_output.py ├── test_init.py ├── test_llm_calls ├── cassettes │ ├── test_anthropic │ │ ├── Test_Anthropic_LLMs.test_claude_21_response_model.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_21_with_async_streaming.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_21_with_streaming.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_5_haiku_with_openai_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_5_haiku_with_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_5_sonnet_20241022_with_openai_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_5_sonnet_20241022_with_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_5_sonnet_with_openai_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_5_sonnet_with_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_7_sonnet_20250219_with_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_7_sonnet_with_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_haiku_with_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_opus_response_model.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_opus_with_openai_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_opus_with_tool_calling.yaml │ │ ├── Test_Anthropic_LLMs.test_claude_3_sonnet_with_openai_tool_calling.yaml │ │ └── Test_Anthropic_LLMs.test_claude_3_sonnet_with_tool_calling.yaml │ ├── test_cohere │ │ ├── Test_Cohere.test_cohere_command_r_plus_response_model.yaml │ │ ├── Test_Cohere.test_cohere_command_r_plus_with_tool_calling.yaml │ │ ├── Test_Cohere.test_cohere_command_r_response_model.yaml │ │ └── Test_Cohere.test_cohere_command_r_with_tool_calling.yaml │ ├── test_mistral │ │ ├── Test_Mistral.test_codestral_response_model.yaml │ │ ├── Test_Mistral.test_mistral_7b_response_model.yaml │ │ ├── Test_Mistral.test_mistral_8x22b_response_model.yaml │ │ ├── Test_Mistral.test_mistral_8x7b_response_model.yaml │ │ ├── Test_Mistral.test_mistral_large_2402_response_model.yaml │ │ ├── Test_Mistral.test_mistral_large_2402_with_openai_tool_calling.yaml │ │ ├── Test_Mistral.test_mistral_large_2402_with_tool_calling.yaml │ │ ├── Test_Mistral.test_mistral_large_2407_response_model.yaml │ │ ├── Test_Mistral.test_mistral_large_2407_with_openai_tool_calling.yaml │ │ ├── Test_Mistral.test_mistral_large_2407_with_tool_calling.yaml │ │ ├── Test_Mistral.test_mistral_large_response_model.yaml │ │ ├── Test_Mistral.test_mistral_large_with_openai_tool_calling.yaml │ │ ├── Test_Mistral.test_mistral_large_with_tool_calling.yaml │ │ ├── Test_Mistral.test_mistral_medium_response_model.yaml │ │ ├── Test_Mistral.test_mistral_nemo_response_model.yaml │ │ ├── Test_Mistral.test_mistral_nemo_with_openai_tool_calling.yaml │ │ ├── Test_Mistral.test_mistral_nemo_with_tool_calling.yaml │ │ ├── Test_Mistral.test_mistral_small_response_model.yaml │ │ ├── Test_Mistral.test_mistral_small_with_openai_tool_calling.yaml │ │ └── Test_Mistral.test_mistral_small_with_tool_calling.yaml │ ├── test_openai │ │ ├── Test_OpenAI.test_async_streaming[provider0].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider10].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider11].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider12].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider13].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider14].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider15].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider16].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider17].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider18].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider19].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider1].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider20].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider2].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider3].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider4].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider5].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider6].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider7].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider8].yaml │ │ ├── Test_OpenAI.test_async_streaming[provider9].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider0].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider10].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider11].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider12].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider13].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider14].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider15].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider16].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider1].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider2].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider3].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider4].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider5].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider6].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider7].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider8].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_2025_02_27_with_tool_calling[provider9].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider0].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider10].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider11].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider12].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider13].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider14].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider15].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider16].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider1].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider2].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider3].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider4].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider5].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider6].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider7].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider8].yaml │ │ ├── Test_OpenAI.test_gpt_4_5_preview_with_tool_calling[provider9].yaml │ │ ├── Test_OpenAI.test_response_model[provider0].yaml │ │ ├── Test_OpenAI.test_response_model[provider10].yaml │ │ ├── Test_OpenAI.test_response_model[provider11].yaml │ │ ├── Test_OpenAI.test_response_model[provider12].yaml │ │ ├── Test_OpenAI.test_response_model[provider13].yaml │ │ ├── Test_OpenAI.test_response_model[provider14].yaml │ │ ├── Test_OpenAI.test_response_model[provider15].yaml │ │ ├── Test_OpenAI.test_response_model[provider16].yaml │ │ ├── Test_OpenAI.test_response_model[provider17].yaml │ │ ├── Test_OpenAI.test_response_model[provider18].yaml │ │ ├── Test_OpenAI.test_response_model[provider19].yaml │ │ ├── Test_OpenAI.test_response_model[provider1].yaml │ │ ├── Test_OpenAI.test_response_model[provider20].yaml │ │ ├── Test_OpenAI.test_response_model[provider2].yaml │ │ ├── Test_OpenAI.test_response_model[provider3].yaml │ │ ├── Test_OpenAI.test_response_model[provider4].yaml │ │ ├── Test_OpenAI.test_response_model[provider5].yaml │ │ ├── Test_OpenAI.test_response_model[provider6].yaml │ │ ├── Test_OpenAI.test_response_model[provider7].yaml │ │ ├── Test_OpenAI.test_response_model[provider8].yaml │ │ ├── Test_OpenAI.test_response_model[provider9].yaml │ │ ├── Test_OpenAI.test_streaming[provider0].yaml │ │ ├── Test_OpenAI.test_streaming[provider10].yaml │ │ ├── Test_OpenAI.test_streaming[provider11].yaml │ │ ├── Test_OpenAI.test_streaming[provider12].yaml │ │ ├── Test_OpenAI.test_streaming[provider13].yaml │ │ ├── Test_OpenAI.test_streaming[provider14].yaml │ │ ├── Test_OpenAI.test_streaming[provider15].yaml │ │ ├── Test_OpenAI.test_streaming[provider16].yaml │ │ ├── Test_OpenAI.test_streaming[provider17].yaml │ │ ├── Test_OpenAI.test_streaming[provider18].yaml │ │ ├── Test_OpenAI.test_streaming[provider19].yaml │ │ ├── Test_OpenAI.test_streaming[provider1].yaml │ │ ├── Test_OpenAI.test_streaming[provider20].yaml │ │ ├── Test_OpenAI.test_streaming[provider2].yaml │ │ ├── Test_OpenAI.test_streaming[provider3].yaml │ │ ├── Test_OpenAI.test_streaming[provider4].yaml │ │ ├── Test_OpenAI.test_streaming[provider5].yaml │ │ ├── Test_OpenAI.test_streaming[provider6].yaml │ │ ├── Test_OpenAI.test_streaming[provider7].yaml │ │ ├── Test_OpenAI.test_streaming[provider8].yaml │ │ ├── Test_OpenAI.test_streaming[provider9].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider0].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider10].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider11].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider12].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider13].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider14].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider15].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider16].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider17].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider18].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider19].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider1].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider20].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider2].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider3].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider4].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider5].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider6].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider7].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider8].yaml │ │ ├── Test_OpenAI.test_streaming_use_stop[provider9].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider0].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider10].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider11].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider12].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider13].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider14].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider15].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider16].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider17].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider18].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider19].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider1].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider2].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider3].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider4].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider5].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider6].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider7].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider8].yaml │ │ ├── Test_OpenAI.test_with_openai_tool_calling[provider9].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider0].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider10].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider11].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider12].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider13].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider14].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider15].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider16].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider17].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider18].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider19].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider1].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider2].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider3].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider4].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider5].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider6].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider7].yaml │ │ ├── Test_OpenAI.test_with_tool_calling[provider8].yaml │ │ └── Test_OpenAI.test_with_tool_calling[provider9].yaml │ ├── test_openai_o1 │ │ ├── test_o1_with_system_prompt[provider0].yaml │ │ ├── test_o1_with_system_prompt[provider1].yaml │ │ ├── test_o1_with_system_prompt[provider2].yaml │ │ └── test_o1_with_system_prompt[provider3].yaml │ ├── test_perplexity │ │ ├── Test_Perplexity_LLMs.test_llama_3_1_sonar_large_128k_online.yaml │ │ └── Test_Perplexity_LLMs.test_sonar.yaml │ └── test_togetherai │ │ ├── Test_TogetherAI_LLMs.test_deepseek_r1.yaml │ │ ├── Test_TogetherAI_LLMs.test_llama_3_1_405b_instruct_turbo.yaml │ │ ├── Test_TogetherAI_LLMs.test_llama_3_1_70b_instruct_turbo.yaml │ │ ├── Test_TogetherAI_LLMs.test_llama_3_1_8b_instruct_turbo.yaml │ │ ├── Test_TogetherAI_LLMs.test_llama_3_70b_chat_hf.yaml │ │ ├── Test_TogetherAI_LLMs.test_llama_3_8b_chat_hf.yaml │ │ ├── Test_TogetherAI_LLMs.test_mistral_7b_instruct_v02.yaml │ │ ├── Test_TogetherAI_LLMs.test_mixtral_8x7b_instruct_v01.yaml │ │ └── Test_TogetherAI_LLMs.test_qwen2_72b.yaml ├── test_anthropic.py ├── test_cohere.py ├── test_google.py ├── test_mistral.py ├── test_openai.py ├── test_openai_o1.py ├── test_perplexity.py ├── test_replicate.py └── test_togetherai.py ├── test_toolkit ├── cassettes │ ├── test_async_completion_notdiamond.yaml │ ├── test_async_completion_notdiamond_stream.yaml │ ├── test_completion_notdiamond.yaml │ ├── test_completion_notdiamond_stream.yaml │ ├── test_completion_notdiamond_tool_calling.yaml │ ├── test_litellm │ │ ├── test_async_completion_notdiamond.yaml │ │ ├── test_async_completion_notdiamond_stream.yaml │ │ ├── test_completion_notdiamond_stream.yaml │ │ └── test_completion_notdiamond_tool_calling.yaml │ ├── test_openai_client │ │ ├── test_async_openai_chat_completions_create.yaml │ │ ├── test_async_openai_chat_completions_create_stream.yaml │ │ ├── test_async_openai_create.yaml │ │ ├── test_async_openai_create_default_models.yaml │ │ ├── test_openai_chat_completions_create.yaml │ │ ├── test_openai_chat_completions_create_stream.yaml │ │ ├── test_openai_create.yaml │ │ └── test_openai_create_default_models.yaml │ └── test_retry │ │ ├── test_async_multi_model_max_retries_config.yaml │ │ ├── test_async_multi_model_model_messages_config.yaml │ │ ├── test_async_multi_model_multi_provider_azure_error.yaml │ │ ├── test_async_multi_model_multi_provider_load_balance.yaml │ │ ├── test_async_multi_model_timeout_config.yaml │ │ ├── test_multi_model_backoff_config.yaml │ │ ├── test_multi_model_max_retries_config.yaml │ │ ├── test_multi_model_model_messages_config.yaml │ │ ├── test_multi_model_multi_provider_azure_error.yaml │ │ ├── test_multi_model_multi_provider_load_balance.yaml │ │ ├── test_multi_model_timeout_config.yaml │ │ ├── test_retries.yaml │ │ ├── test_retry_wrapper[anthropic].yaml │ │ ├── test_retry_wrapper[openai].yaml │ │ ├── test_retry_wrapper_async[async-anthropic].yaml │ │ ├── test_retry_wrapper_async[async-openai].yaml │ │ ├── test_retry_wrapper_async_create[async-anthropic].yaml │ │ ├── test_retry_wrapper_async_create[async-azure-openai].yaml │ │ ├── test_retry_wrapper_async_create[async-openai].yaml │ │ ├── test_retry_wrapper_async_stream[async-anthropic].yaml │ │ ├── test_retry_wrapper_async_stream[async-azure-openai].yaml │ │ ├── test_retry_wrapper_async_stream[async-openai].yaml │ │ ├── test_retry_wrapper_create[anthropic].yaml │ │ ├── test_retry_wrapper_create[azure-openai].yaml │ │ ├── test_retry_wrapper_create[openai].yaml │ │ ├── test_retry_wrapper_stream[anthropic].yaml │ │ ├── test_retry_wrapper_stream[azure-openai].yaml │ │ └── test_retry_wrapper_stream[openai].yaml ├── langchain │ ├── cassettes │ │ └── test_integration │ │ │ ├── test_invokable[anthropic-claude-3-5-sonnet-20240620-langchain_anthropic.ChatAnthropic].yaml │ │ │ ├── test_invokable[anthropic-claude-3-haiku-20240307-langchain_anthropic.ChatAnthropic].yaml │ │ │ ├── test_invokable[anthropic-claude-3-opus-20240229-langchain_anthropic.ChatAnthropic].yaml │ │ │ ├── test_invokable[anthropic-claude-3-sonnet-20240229-langchain_anthropic.ChatAnthropic].yaml │ │ │ ├── test_invokable[cohere-command-r-langchain_cohere.ChatCohere].yaml │ │ │ ├── test_invokable[cohere-command-r-plus-langchain_cohere.ChatCohere].yaml │ │ │ ├── test_invokable[google-gemini-1.5-flash-latest-langchain_google_genai.ChatGoogleGenerativeAI].yaml │ │ │ ├── test_invokable[google-gemini-1.5-pro-latest-langchain_google_genai.ChatGoogleGenerativeAI].yaml │ │ │ ├── test_invokable[mistral-codestral-latest-langchain_mistralai.ChatMistralAI].yaml │ │ │ ├── test_invokable[mistral-mistral-large-2402-langchain_mistralai.ChatMistralAI].yaml │ │ │ ├── test_invokable[mistral-mistral-large-2407-langchain_mistralai.ChatMistralAI].yaml │ │ │ ├── test_invokable[mistral-mistral-medium-latest-langchain_mistralai.ChatMistralAI].yaml │ │ │ ├── test_invokable[mistral-mistral-small-latest-langchain_mistralai.ChatMistralAI].yaml │ │ │ ├── test_invokable[mistral-open-mistral-7b-langchain_mistralai.ChatMistralAI].yaml │ │ │ ├── test_invokable[mistral-open-mixtral-8x22b-langchain_mistralai.ChatMistralAI].yaml │ │ │ ├── test_invokable[mistral-open-mixtral-8x7b-langchain_mistralai.ChatMistralAI].yaml │ │ │ ├── test_invokable[openai-gpt-3.5-turbo-0125-langchain_openai.ChatOpenAI].yaml │ │ │ ├── test_invokable[openai-gpt-4-0125-preview-langchain_openai.ChatOpenAI].yaml │ │ │ ├── test_invokable[openai-gpt-4-0613-langchain_openai.ChatOpenAI].yaml │ │ │ ├── test_invokable[openai-gpt-4-1106-preview-langchain_openai.ChatOpenAI].yaml │ │ │ ├── test_invokable[openai-gpt-4-turbo-2024-04-09-langchain_openai.ChatOpenAI].yaml │ │ │ ├── test_invokable[openai-gpt-4o-2024-05-13-langchain_openai.ChatOpenAI].yaml │ │ │ ├── test_invokable[openai-gpt-4o-2024-08-06-langchain_openai.ChatOpenAI].yaml │ │ │ ├── test_invokable[openai-gpt-4o-langchain_openai.ChatOpenAI].yaml │ │ │ ├── test_invokable[openai-gpt-4o-mini-2024-07-18-langchain_openai.ChatOpenAI].yaml │ │ │ ├── test_invokable[togetherai-Llama-3-70b-chat-hf-langchain_together.ChatTogether].yaml │ │ │ ├── test_invokable[togetherai-Llama-3-8b-chat-hf-langchain_together.ChatTogether].yaml │ │ │ ├── test_invokable[togetherai-Meta-Llama-3.1-405B-Instruct-Turbo-langchain_together.ChatTogether].yaml │ │ │ ├── test_invokable[togetherai-Meta-Llama-3.1-70B-Instruct-Turbo-langchain_together.ChatTogether].yaml │ │ │ ├── test_invokable[togetherai-Meta-Llama-3.1-8B-Instruct-Turbo-langchain_together.ChatTogether].yaml │ │ │ ├── test_invokable[togetherai-Mistral-7B-Instruct-v0.2-langchain_together.ChatTogether].yaml │ │ │ ├── test_invokable[togetherai-Mixtral-8x22B-Instruct-v0.1-langchain_together.ChatTogether].yaml │ │ │ ├── test_invokable[togetherai-Mixtral-8x7B-Instruct-v0.1-langchain_together.ChatTogether].yaml │ │ │ ├── test_invokable[togetherai-Qwen2-72B-Instruct-langchain_together.ChatTogether].yaml │ │ │ ├── test_notdiamond_routed_runnable.yaml │ │ │ └── test_notdiamond_routed_runnable_chain.yaml │ ├── test_integration.py │ └── test_unit.py ├── rag │ ├── cassettes │ │ └── test_evaluation │ │ │ └── test_evaluate.yaml │ ├── conftest.py │ ├── test_data_gen.py │ ├── test_evaluation.py │ ├── test_example_workflow.py │ └── test_workflow.py ├── test_custom_router.py ├── test_openai_client.py └── test_retry.py └── test_types.py /.github/actions/set-working-directory/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Set the working directory' 2 | description: 'Sets the environment variable {{ env.WORKING_DIRECTORY }} for use in subsequent actions' 3 | 4 | 5 | inputs: 6 | working_directory: 7 | description: 'Directory to set as env.WORKING_DIRECTORY' 8 | required: false 9 | default: ${{ github.workspace }} 10 | 11 | 12 | runs: 13 | using: "composite" 14 | steps: 15 | - name: set-working-directory 16 | run: | 17 | WORKING_DIRECTORY=${{ inputs.working_directory }} 18 | echo "WORKING_DIRECTORY: $WORKING_DIRECTORY" 19 | echo "WORKING_DIRECTORY=$WORKING_DIRECTORY" >> "$GITHUB_ENV" 20 | shell: bash 21 | -------------------------------------------------------------------------------- /.github/actions/setup-python-env/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Setup python environment' 2 | description: 'Install python and packages into environment using Poetry' 3 | 4 | inputs: 5 | working_directory: 6 | description: 'Directory to run the action' 7 | required: false 8 | default: ${{ github.workspace }} 9 | python_version: 10 | description: 'Python version to use.' 11 | required: false 12 | default: '3.10' 13 | 14 | runs: 15 | using: "composite" 16 | steps: 17 | - name: Install poetry 18 | run: | 19 | python3 -m pip install --upgrade pip 20 | pip3 install poetry==1.7.1 21 | shell: bash 22 | working-directory: ${{ inputs.working_directory }} 23 | - uses: actions/setup-python@v4 24 | with: 25 | cache: 'poetry' 26 | python-version: ${{ inputs.python_version }} 27 | - name: setup venv 28 | run: | 29 | poetry install 30 | echo "$(poetry env info --path)/bin" >> $GITHUB_PATH 31 | shell: bash 32 | working-directory: ${{ inputs.working_directory }} 33 | -------------------------------------------------------------------------------- /.github/workflows/all-sdk-tests.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: notdiamond SDK tests 5 | 6 | on: 7 | pull_request: 8 | push: 9 | branches: 10 | - "main" 11 | 12 | defaults: 13 | run: 14 | working-directory: ./ 15 | 16 | jobs: 17 | build: 18 | runs-on: ubuntu-latest 19 | continue-on-error: true 20 | permissions: 21 | # Gives the action the necessary permissions for publishing new 22 | # comments in pull requests. 23 | pull-requests: write 24 | # Gives the action the necessary permissions for pushing data to the 25 | # python-coverage-comment-action branch, and for editing existing 26 | # comments (to avoid publishing multiple comments in the same PR) 27 | contents: write 28 | 29 | strategy: 30 | matrix: 31 | python-version: ["3.10", "3.11", "3.12"] 32 | steps: 33 | - uses: actions/checkout@v4 34 | - name: Set up Python ${{ matrix.python-version }} 35 | uses: actions/setup-python@v3 36 | with: 37 | python-version: ${{ matrix.python-version }} 38 | - name: Install Poetry 39 | uses: snok/install-poetry@v1 40 | with: 41 | virtualenvs-create: true 42 | virtualenvs-in-project: true 43 | installer-parallel: true 44 | - name: Load cached venv 45 | id: cached-poetry-dependencies 46 | uses: actions/cache@v4 47 | with: 48 | path: .venv 49 | key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }} 50 | - name: use correct env 51 | run: poetry env use ${{ matrix.python-version }} 52 | - name: Install dependencies if venv is not cached 53 | if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' 54 | run: poetry install -E "create rag openai" --with test && poetry run python -m nltk.downloader all 55 | - name: Test with pytest 56 | env: 57 | NOTDIAMOND_API_KEY: ${{ secrets.NOTDIAMOND_API_KEY }} 58 | OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} 59 | OPENAI_API_VERSION: ${{ secrets.OPENAI_API_VERSION }} 60 | AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_API_KEY }} 61 | AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }} 62 | AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} 63 | ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} 64 | GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} 65 | COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} 66 | MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} 67 | TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} 68 | REPLICATE_API_KEY: ${{ secrets.REPLICATE_API_KEY }} 69 | PPLX_API_KEY: ${{ secrets.PPLX_API_KEY }} 70 | OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} 71 | NOTDIAMOND_API_URL: ${{ vars.NOTDIAMOND_API_URL }} 72 | run: | 73 | PYTEST_ADDOPTS="--color=yes" poetry run coverage run --branch -m pytest -v tests/ 74 | poetry run coverage report -m 75 | - name: coverage badge + comment 76 | uses: py-cov-action/python-coverage-comment-action@v3 77 | with: 78 | GITHUB_TOKEN: ${{ github.token }} 79 | # If the coverage percentage is above or equal to this value, the badge will be green. 80 | MINIMUM_GREEN: 80 81 | MINIMUM_ORANGE: 50 82 | COVERAGE_PATH: ./ 83 | - name: Store Pull Request comment to be posted 84 | uses: actions/upload-artifact@v4 85 | if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true' 86 | with: 87 | # If you use a different name, update COMMENT_ARTIFACT_NAME accordingly 88 | name: python-coverage-comment-action 89 | # If you use a different name, update COMMENT_FILENAME accordingly 90 | path: python-coverage-comment-action.txt 91 | -------------------------------------------------------------------------------- /.github/workflows/build_docs.yml: -------------------------------------------------------------------------------- 1 | # Simple workflow for deploying static content to GitHub Pages 2 | name: Build docs based on Sphinx 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | push: 7 | branches: ["main"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | environment: 15 | name: build-docs 16 | runs-on: ubuntu-22.04 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | 21 | - name: Set working directory 22 | uses: ./.github/actions/set-working-directory 23 | with: 24 | working_directory: "${{ github.workspace }}" 25 | 26 | - name: Setup python environment 27 | uses: ./.github/actions/setup-python-env 28 | with: 29 | working_directory: "${{ env.WORKING_DIRECTORY }}" 30 | 31 | - name: Build docs 32 | run: | 33 | cd docs 34 | make html 35 | 36 | - uses: EndBug/add-and-commit@v9.1.1 37 | with: 38 | add: "${{ env.WORKING_DIRECTORY }}/docs/_build/html" 39 | default_author: github_actor 40 | fetch: true 41 | message: "[skip ci] Update docs" 42 | pathspec_error_handling: ignore 43 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | # Write a test coverage comment to each pull request which reports coverage 2 | # changes due to the PR. 3 | name: Post coverage comment 4 | 5 | on: 6 | workflow_run: 7 | workflows: ["notdiamond SDK tests"] 8 | types: 9 | - completed 10 | 11 | jobs: 12 | coverage-comment: 13 | name: Run tests & display coverage 14 | runs-on: ubuntu-latest 15 | if: github.event.workflow_run.conclusion == 'success' 16 | permissions: 17 | # Gives the action the necessary permissions for publishing new 18 | # comments in pull requests. 19 | pull-requests: write 20 | # Gives the action the necessary permissions for editing existing 21 | # comments (to avoid publishing multiple comments in the same PR) 22 | contents: write 23 | # Gives the action the necessary permissions for looking up the 24 | # workflow that launched this workflow, and download the related 25 | # artifact that contains the comment to be published 26 | actions: read 27 | steps: 28 | # DO NOT run actions/checkout here, for security reasons 29 | # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ 30 | - name: Post comment 31 | uses: py-cov-action/python-coverage-comment-action@v3 32 | with: 33 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 34 | GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }} 35 | # Update those if you changed the default values: 36 | # COMMENT_ARTIFACT_NAME: python-coverage-comment-action 37 | # COMMENT_FILENAME: python-coverage-comment-action.txt 38 | # If the coverage percentage is above or equal to this value, the badge will be green. 39 | MINIMUM_GREEN: 80 40 | # Same with orange. Below is red. 41 | MINIMUM_ORANGE: 50 42 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: nd-python release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v[0-9]+.[0-9]+.[0-9]+' 7 | 8 | permissions: 9 | contents: write 10 | id-token: write 11 | 12 | jobs: 13 | release-ndserver: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: Set up Python 3.11 18 | id: setup-python 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: "3.11" 22 | - name: Install Poetry 23 | uses: snok/install-poetry@v1 24 | with: 25 | virtualenvs-create: true 26 | virtualenvs-in-project: true 27 | installer-parallel: true 28 | - name: poetry build 29 | run: poetry build 30 | - name: Release 31 | uses: softprops/action-gh-release@v2 32 | if: startsWith(github.ref, 'refs/tags/') 33 | - name: pypi 34 | uses: pypa/gh-action-pypi-publish@release/v1 35 | with: 36 | user: __token__ 37 | password: ${{ secrets.PYPI_API_TOKEN }} 38 | -------------------------------------------------------------------------------- /.github/workflows/static_docs.yml: -------------------------------------------------------------------------------- 1 | # Simple workflow for deploying static content to GitHub Pages 2 | name: Deploy docs to GitHub Pages 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | release: 7 | types: [published] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 20 | concurrency: 21 | group: "pages" 22 | cancel-in-progress: false 23 | 24 | jobs: 25 | deploy: 26 | environment: 27 | name: github-pages 28 | url: ${{ steps.deployment.outputs.page_url }} 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Checkout 32 | uses: actions/checkout@v4 33 | - name: Setup Pages 34 | uses: actions/configure-pages@v5 35 | - name: Upload artifact 36 | uses: actions/upload-pages-artifact@v3 37 | with: 38 | # Upload entire repository 39 | path: './docs/_build/html' 40 | - name: Deploy to GitHub Pages 41 | id: deployment 42 | uses: actions/deploy-pages@v4 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | **/.DS_Store 3 | .env 4 | .envrc 5 | .python-version 6 | dist/ 7 | !dist/.gitkeep 8 | /dist 9 | .vscode/ 10 | venv/ 11 | .venv/ 12 | .idea/ 13 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v4.4.0 4 | hooks: 5 | - id: check-yaml 6 | - id: end-of-file-fixer 7 | - id: trailing-whitespace 8 | 9 | - repo: https://github.com/timothycrosley/isort 10 | rev: 5.13.2 11 | hooks: 12 | - id: isort 13 | args: ["--profile", "black", "--filter-files"] 14 | exclude: ^(cookbooks|docs) 15 | 16 | - repo: https://github.com/psf/black 17 | rev: 23.3.0 18 | hooks: 19 | - id: black 20 | exclude: ^(cookbooks|docs) 21 | 22 | - repo: https://github.com/PyCQA/flake8 23 | rev: 7.0.0 24 | hooks: 25 | - id: flake8 26 | args: [--max-line-length=130] 27 | exclude: ^(cookbooks|docs|tests/conftest.py) 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # notdiamond-python 2 | 3 | Not Diamond is an AI model router that automatically determines which LLM is best-suited to respond to any query, improving LLM output quality by combining multiple LLMs into a **meta-model** that learns when to call each LLM. 4 | 5 | ## Key features 6 | 7 | - **[Maximize output quality](https://notdiamond.readme.io/docs/quickstart)**: Not Diamond [outperforms every foundation model](https://notdiamond.readme.io/docs/benchmark-performance) on major evaluation benchmarks by always calling the best model for every prompt. 8 | - **[Reduce cost and latency](https://notdiamond.readme.io/docs/cost-and-latency-tradeoffs)**: Not Diamond lets you define intelligent cost and latency tradeoffs to efficiently leverage smaller and cheaper models without degrading quality. 9 | - **[Train your own router](https://notdiamond.readme.io/docs/router-training-quickstart)**: Not Diamond lets you train your own custom routers optimized to your data and use case. 10 | - **[Python](https://python.notdiamond.ai/), [TypeScript](https://www.npmjs.com/package/notdiamond), and [REST API](https://notdiamond.readme.io/reference/api-introduction) support**: Not Diamond works across a variety of stacks. 11 | 12 | ## Installation 13 | 14 | **Python**: Requires **Python 3.10+**. It's recommended that you create and activate a [virtualenv](https://virtualenv.pypa.io/en/latest/) prior to installing the package. For this example, we'll be installing the optional additional `create` dependencies, which you can learn more about [here](https://notdiamond.readme.io/docs/model_select-vs-create). 15 | 16 | ```shell 17 | pip install notdiamond[create] 18 | ``` 19 | 20 | ### Apple Silicon 21 | 22 | Make sure to install `libmagic` if you intend to use the `create` or `rag` extras: 23 | 24 | ```bash 25 | brew install libmagic 26 | ``` 27 | 28 | ## Setting up 29 | 30 | Create a `.env` file with your [Not Diamond API key](https://app.notdiamond.ai/keys) and the [API keys of the models](https://notdiamond.readme.io/docs/api-keys) you want to route between: 31 | 32 | ```shell 33 | NOTDIAMOND_API_KEY = "YOUR_NOTDIAMOND_API_KEY" 34 | OPENAI_API_KEY = "YOUR_OPENAI_API_KEY" 35 | ANTHROPIC_API_KEY = "YOUR_ANTHROPIC_API_KEY" 36 | ``` 37 | 38 | ## Sending your first Not Diamond API request 39 | 40 | Create a new file in the same directory as your `.env` file and copy and run the code below (you can toggle between Python and TypeScript in the top left of the code block): 41 | 42 | ```python 43 | from notdiamond import NotDiamond 44 | 45 | # Define the Not Diamond routing client 46 | client = NotDiamond() 47 | 48 | # The best LLM is determined by Not Diamond based on the messages and specified models 49 | result, session_id, provider = client.chat.completions.create( 50 | messages=[ 51 | {"role": "system", "content": "You are a helpful assistant."}, 52 | {"role": "user", "content": "Concisely explain merge sort."} # Adjust as desired 53 | ], 54 | model=['openai/gpt-3.5-turbo', 'openai/gpt-4o', 'anthropic/claude-3-5-sonnet-20240620'] 55 | ) 56 | 57 | print("ND session ID: ", session_id) # A unique ID of Not Diamond's recommendation 58 | print("LLM called: ", provider.model) # The LLM routed to 59 | print("LLM output: ", result.content) # The LLM response 60 | ``` 61 | -------------------------------------------------------------------------------- /build/release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | EXPECTED_ARGS=1 4 | if [ $# -ne $EXPECTED_ARGS ]; then 5 | echo "Invalid number of arguments. Expected: $EXPECTED_ARGS" 6 | exit 1 7 | fi 8 | 9 | VALID_VERSIONS=("major" "minor" "patch") 10 | if [[ ! " ${VALID_VERSIONS[@]} " =~ " $1 " ]]; then 11 | echo "Invalid version type. Expected one of: ${VALID_VERSIONS[*]}" 12 | exit 1 13 | fi 14 | 15 | git checkout main && git pull 16 | poetry version $1 17 | TAG_NAME=v$(poetry version -s) 18 | git add pyproject.toml 19 | git commit -m "Bump version to $TAG_NAME" 20 | git push origin main 21 | 22 | git tag $TAG_NAME && git push origin tag $TAG_NAME 23 | -------------------------------------------------------------------------------- /dist/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/dist/.gitkeep -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/_build/doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/environment.pickle -------------------------------------------------------------------------------- /docs/_build/doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/index.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/examples.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/examples.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/intro.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/intro.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/modules.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/modules.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/notdiamond.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/notdiamond.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/notdiamond.llms.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/notdiamond.llms.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/notdiamond.metrics.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/notdiamond.metrics.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/notdiamond.prompts.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/notdiamond.prompts.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/notdiamond.toolkit.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/notdiamond.toolkit.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/notdiamond.toolkit.litellm.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/notdiamond.toolkit.litellm.doctree -------------------------------------------------------------------------------- /docs/_build/doctrees/source/notdiamond.toolkit.rag.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/doctrees/source/notdiamond.toolkit.rag.doctree -------------------------------------------------------------------------------- /docs/_build/html/.buildinfo: -------------------------------------------------------------------------------- 1 | # Sphinx build info version 1 2 | # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. 3 | config: 204a5c462923e8038e5918346d8d885f 4 | tags: 645f666f9bcd5a90fca523b33c5a78b7 5 | -------------------------------------------------------------------------------- /docs/_build/html/.doctrees/environment.pickle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/.doctrees/environment.pickle -------------------------------------------------------------------------------- /docs/_build/html/.doctrees/index.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/.doctrees/index.doctree -------------------------------------------------------------------------------- /docs/_build/html/.doctrees/source/intro.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/.doctrees/source/intro.doctree -------------------------------------------------------------------------------- /docs/_build/html/.doctrees/source/notdiamond.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/.doctrees/source/notdiamond.doctree -------------------------------------------------------------------------------- /docs/_build/html/.doctrees/source/notdiamond.llms.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/.doctrees/source/notdiamond.llms.doctree -------------------------------------------------------------------------------- /docs/_build/html/.doctrees/source/notdiamond.metrics.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/.doctrees/source/notdiamond.metrics.doctree -------------------------------------------------------------------------------- /docs/_build/html/.doctrees/source/notdiamond.toolkit.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/.doctrees/source/notdiamond.toolkit.doctree -------------------------------------------------------------------------------- /docs/_build/html/.doctrees/source/notdiamond.toolkit.rag.doctree: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/.doctrees/source/notdiamond.toolkit.rag.doctree -------------------------------------------------------------------------------- /docs/_build/html/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/.nojekyll -------------------------------------------------------------------------------- /docs/_build/html/_static/css/badge_only.css: -------------------------------------------------------------------------------- 1 | .clearfix{*zoom:1}.clearfix:after,.clearfix:before{display:table;content:""}.clearfix:after{clear:both}@font-face{font-family:FontAwesome;font-style:normal;font-weight:400;src:url(fonts/fontawesome-webfont.eot?674f50d287a8c48dc19ba404d20fe713?#iefix) format("embedded-opentype"),url(fonts/fontawesome-webfont.woff2?af7ae505a9eed503f8b8e6982036873e) format("woff2"),url(fonts/fontawesome-webfont.woff?fee66e712a8a08eef5805a46892932ad) format("woff"),url(fonts/fontawesome-webfont.ttf?b06871f281fee6b241d60582ae9369b9) format("truetype"),url(fonts/fontawesome-webfont.svg?912ec66d7572ff821749319396470bde#FontAwesome) format("svg")}.fa:before{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1}.fa:before,a .fa{text-decoration:inherit}.fa:before,a .fa,li .fa{display:inline-block}li .fa-large:before{width:1.875em}ul.fas{list-style-type:none;margin-left:2em;text-indent:-.8em}ul.fas li .fa{width:.8em}ul.fas li .fa-large:before{vertical-align:baseline}.fa-book:before,.icon-book:before{content:"\f02d"}.fa-caret-down:before,.icon-caret-down:before{content:"\f0d7"}.fa-caret-up:before,.icon-caret-up:before{content:"\f0d8"}.fa-caret-left:before,.icon-caret-left:before{content:"\f0d9"}.fa-caret-right:before,.icon-caret-right:before{content:"\f0da"}.rst-versions{position:fixed;bottom:0;left:0;width:300px;color:#fcfcfc;background:#1f1d1d;font-family:Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;z-index:400}.rst-versions a{color:#2980b9;text-decoration:none}.rst-versions .rst-badge-small{display:none}.rst-versions .rst-current-version{padding:12px;background-color:#272525;display:block;text-align:right;font-size:90%;cursor:pointer;color:#27ae60}.rst-versions .rst-current-version:after{clear:both;content:"";display:block}.rst-versions .rst-current-version .fa{color:#fcfcfc}.rst-versions .rst-current-version .fa-book,.rst-versions .rst-current-version .icon-book{float:left}.rst-versions .rst-current-version.rst-out-of-date{background-color:#e74c3c;color:#fff}.rst-versions .rst-current-version.rst-active-old-version{background-color:#f1c40f;color:#000}.rst-versions.shift-up{height:auto;max-height:100%;overflow-y:scroll}.rst-versions.shift-up .rst-other-versions{display:block}.rst-versions .rst-other-versions{font-size:90%;padding:12px;color:grey;display:none}.rst-versions .rst-other-versions hr{display:block;height:1px;border:0;margin:20px 0;padding:0;border-top:1px solid #413d3d}.rst-versions .rst-other-versions dd{display:inline-block;margin:0}.rst-versions .rst-other-versions dd a{display:inline-block;padding:6px;color:#fcfcfc}.rst-versions.rst-badge{width:auto;bottom:20px;right:20px;left:auto;border:none;max-width:300px;max-height:90%}.rst-versions.rst-badge .fa-book,.rst-versions.rst-badge .icon-book{float:none;line-height:30px}.rst-versions.rst-badge.shift-up .rst-current-version{text-align:right}.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{float:left}.rst-versions.rst-badge>.rst-current-version{width:auto;height:30px;line-height:30px;padding:0 6px;display:block;text-align:center}@media screen and (max-width:768px){.rst-versions{width:85%;display:none}.rst-versions.shift{display:block}} -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/Roboto-Slab-Bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/Roboto-Slab-Bold.woff -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/Roboto-Slab-Bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/Roboto-Slab-Bold.woff2 -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/Roboto-Slab-Regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/Roboto-Slab-Regular.woff -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/Roboto-Slab-Regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/Roboto-Slab-Regular.woff2 -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/lato-bold-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/lato-bold-italic.woff -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/lato-bold-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/lato-bold-italic.woff2 -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/lato-bold.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/lato-bold.woff -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/lato-bold.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/lato-bold.woff2 -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/lato-normal-italic.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/lato-normal-italic.woff -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/lato-normal-italic.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/lato-normal-italic.woff2 -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/lato-normal.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/lato-normal.woff -------------------------------------------------------------------------------- /docs/_build/html/_static/css/fonts/lato-normal.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/css/fonts/lato-normal.woff2 -------------------------------------------------------------------------------- /docs/_build/html/_static/documentation_options.js: -------------------------------------------------------------------------------- 1 | const DOCUMENTATION_OPTIONS = { 2 | VERSION: '0.4.4', 3 | LANGUAGE: 'en', 4 | COLLAPSE_INDEX: false, 5 | BUILDER: 'html', 6 | FILE_SUFFIX: '.html', 7 | LINK_SUFFIX: '.html', 8 | HAS_SOURCE: false, 9 | SOURCELINK_SUFFIX: '.txt', 10 | NAVIGATION_WITH_KEYS: false, 11 | SHOW_SEARCH_SUMMARY: true, 12 | ENABLE_SEARCH_SHORTCUTS: true, 13 | }; -------------------------------------------------------------------------------- /docs/_build/html/_static/file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/file.png -------------------------------------------------------------------------------- /docs/_build/html/_static/js/badge_only.js: -------------------------------------------------------------------------------- 1 | !function(e){var t={};function r(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,r),o.l=!0,o.exports}r.m=e,r.c=t,r.d=function(e,t,n){r.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:n})},r.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.t=function(e,t){if(1&t&&(e=r(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var n=Object.create(null);if(r.r(n),Object.defineProperty(n,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var o in e)r.d(n,o,function(t){return e[t]}.bind(null,o));return n},r.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(t,"a",t),t},r.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r.p="",r(r.s=4)}({4:function(e,t,r){}}); -------------------------------------------------------------------------------- /docs/_build/html/_static/js/html5shiv.min.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @preserve HTML5 Shiv 3.7.3 | @afarkas @jdalton @jon_neal @rem | MIT/GPL2 Licensed 3 | */ 4 | !function(a,b){function c(a,b){var c=a.createElement("p"),d=a.getElementsByTagName("head")[0]||a.documentElement;return c.innerHTML="x",d.insertBefore(c.lastChild,d.firstChild)}function d(){var a=t.elements;return"string"==typeof a?a.split(" "):a}function e(a,b){var c=t.elements;"string"!=typeof c&&(c=c.join(" ")),"string"!=typeof a&&(a=a.join(" ")),t.elements=c+" "+a,j(b)}function f(a){var b=s[a[q]];return b||(b={},r++,a[q]=r,s[r]=b),b}function g(a,c,d){if(c||(c=b),l)return c.createElement(a);d||(d=f(c));var e;return e=d.cache[a]?d.cache[a].cloneNode():p.test(a)?(d.cache[a]=d.createElem(a)).cloneNode():d.createElem(a),!e.canHaveChildren||o.test(a)||e.tagUrn?e:d.frag.appendChild(e)}function h(a,c){if(a||(a=b),l)return a.createDocumentFragment();c=c||f(a);for(var e=c.frag.cloneNode(),g=0,h=d(),i=h.length;i>g;g++)e.createElement(h[g]);return e}function i(a,b){b.cache||(b.cache={},b.createElem=a.createElement,b.createFrag=a.createDocumentFragment,b.frag=b.createFrag()),a.createElement=function(c){return t.shivMethods?g(c,a,b):b.createElem(c)},a.createDocumentFragment=Function("h,f","return function(){var n=f.cloneNode(),c=n.createElement;h.shivMethods&&("+d().join().replace(/[\w\-:]+/g,function(a){return b.createElem(a),b.frag.createElement(a),'c("'+a+'")'})+");return n}")(t,b.frag)}function j(a){a||(a=b);var d=f(a);return!t.shivCSS||k||d.hasCSS||(d.hasCSS=!!c(a,"article,aside,dialog,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}mark{background:#FF0;color:#000}template{display:none}")),l||i(a,d),a}var k,l,m="3.7.3-pre",n=a.html5||{},o=/^<|^(?:button|map|select|textarea|object|iframe|option|optgroup)$/i,p=/^(?:a|b|code|div|fieldset|h1|h2|h3|h4|h5|h6|i|label|li|ol|p|q|span|strong|style|table|tbody|td|th|tr|ul)$/i,q="_html5shiv",r=0,s={};!function(){try{var a=b.createElement("a");a.innerHTML="",k="hidden"in a,l=1==a.childNodes.length||function(){b.createElement("a");var a=b.createDocumentFragment();return"undefined"==typeof a.cloneNode||"undefined"==typeof a.createDocumentFragment||"undefined"==typeof a.createElement}()}catch(c){k=!0,l=!0}}();var t={elements:n.elements||"abbr article aside audio bdi canvas data datalist details dialog figcaption figure footer header hgroup main mark meter nav output picture progress section summary template time video",version:m,shivCSS:n.shivCSS!==!1,supportsUnknownElements:l,shivMethods:n.shivMethods!==!1,type:"default",shivDocument:j,createElement:g,createDocumentFragment:h,addElements:e};a.html5=t,j(b),"object"==typeof module&&module.exports&&(module.exports=t)}("undefined"!=typeof window?window:this,document); -------------------------------------------------------------------------------- /docs/_build/html/_static/minus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/minus.png -------------------------------------------------------------------------------- /docs/_build/html/_static/plus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/_static/plus.png -------------------------------------------------------------------------------- /docs/_build/html/objects.inv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/docs/_build/html/objects.inv -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from pathlib import Path 4 | 5 | import toml 6 | 7 | # Configuration file for the Sphinx documentation builder. 8 | # 9 | # For the full list of built-in configuration values, see the documentation: 10 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 11 | 12 | 13 | _DIR = Path(__file__).parent.absolute() 14 | with (_DIR.parents[0] / "pyproject.toml").open("r") as f: 15 | data = toml.load(f) 16 | 17 | # -- Project information ----------------------------------------------------- 18 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 19 | 20 | project = "NotDiamond" 21 | copyright = "2024, NotDiamond" 22 | author = "NotDiamond" 23 | 24 | version = data["tool"]["poetry"]["version"] 25 | release = version 26 | 27 | html_title = project + " " + version 28 | html_last_updated_fmt = "%b %d, %Y" 29 | html_show_sourcelink = False 30 | 31 | sys.path.insert(0, os.path.abspath("..")) 32 | 33 | # -- General configuration --------------------------------------------------- 34 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 35 | 36 | extensions = [ 37 | "sphinx.ext.autodoc", 38 | "sphinx.ext.doctest", 39 | "sphinx.ext.intersphinx", 40 | "sphinx.ext.todo", 41 | "sphinx.ext.coverage", 42 | "sphinx.ext.mathjax", 43 | "sphinx.ext.ifconfig", 44 | "sphinx.ext.viewcode", 45 | "sphinx.ext.githubpages", 46 | "sphinx.ext.napoleon", 47 | ] 48 | 49 | # Napoleon settings 50 | napoleon_google_docstring = True 51 | napoleon_numpy_docstring = True 52 | napoleon_include_init_with_doc = True 53 | napoleon_include_private_with_doc = True 54 | napoleon_include_special_with_doc = True 55 | napoleon_use_admonition_for_examples = False 56 | napoleon_use_admonition_for_notes = False 57 | napoleon_use_admonition_for_references = False 58 | napoleon_use_ivar = False 59 | napoleon_use_param = True 60 | napoleon_use_rtype = True 61 | autodoc_pydantic_model_show_json = False 62 | autodoc_pydantic_field_list_validators = False 63 | autodoc_pydantic_config_members = False 64 | autodoc_pydantic_model_show_config_summary = False 65 | autodoc_pydantic_model_show_validator_members = False 66 | autodoc_pydantic_model_show_validator_summary = False 67 | autodoc_pydantic_model_signature_prefix = "class" 68 | autodoc_pydantic_field_signature_prefix = "param" 69 | autodoc_member_order = "groupwise" 70 | autoclass_content = "both" 71 | autodoc_typehints_format = "short" 72 | autodoc_typehints = "both" 73 | 74 | templates_path = ["_templates"] 75 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 76 | 77 | # List of patterns, relative to source directory, that match files and 78 | # directories to ignore when looking for source files. 79 | # This pattern also affects html_static_path and html_extra_path. 80 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 81 | 82 | 83 | # -- Options for HTML output ------------------------------------------------- 84 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 85 | 86 | html_theme = "sphinx_rtd_theme" 87 | html_static_path = ["_static"] 88 | html_use_index = False 89 | 90 | # generate autosummary even if no references 91 | autosummary_generate = True 92 | 93 | html_copy_source = False 94 | html_show_sourcelink = False 95 | 96 | autodoc_mock_imports = ["notdiamond.toolkit._retry"] 97 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. NotDiamond documentation master file, created by 2 | sphinx-quickstart on Wed Apr 24 11:09:36 2024. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ======================================== 7 | Not Diamond Python library documentation 8 | ======================================== 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | :caption: Contents: 13 | 14 | source/intro.rst 15 | source/notdiamond.rst 16 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/source/intro.rst: -------------------------------------------------------------------------------- 1 | Getting started with Not Diamond 2 | ================================ 3 | 4 | Not Diamond is an AI model router that automatically determines which LLM is best-suited to respond to any query, improving LLM output quality by combining multiple LLMs into a **meta-model** that learns when to call each LLM. 5 | 6 | ============ 7 | Key features 8 | ============ 9 | 10 | - `Maximize output quality `_: Not Diamond `outperforms every foundation model `_ on major evaluation benchmarks by always calling the best model for every prompt. 11 | - `Reduce cost and latency `_: Not Diamond lets you define intelligent cost and latency tradeoffs to efficiently leverage smaller and cheaper models without degrading quality. 12 | - `Train your own router `_: Not Diamond lets you train your own custom routers optimized to your data and use case. 13 | - `Python `_, `TypeScript `_, and `REST API `_ support: Not Diamond works across a variety of stacks. 14 | 15 | ============ 16 | Installation 17 | ============ 18 | 19 | **Python**: Requires **Python 3.10+**. It’s recommended that you create and activate a `virtualenv `_ prior to installing the package. For this example, we'll be installing the optional additional `create` dependencies, which you can learn more about `here `_. 20 | 21 | .. code:: shell 22 | 23 | pip install notdiamond[create] 24 | 25 | ========== 26 | Setting up 27 | ========== 28 | 29 | Create a `.env` file with your `Not Diamond API key `_ and the `API keys of the models `_ you want to route between: 30 | 31 | .. code:: shell 32 | 33 | NOTDIAMOND_API_KEY = "YOUR_NOTDIAMOND_API_KEY" 34 | OPENAI_API_KEY = "YOUR_OPENAI_API_KEY" 35 | ANTHROPIC_API_KEY = "YOUR_ANTHROPIC_API_KEY" 36 | 37 | ========================================== 38 | Sending your first Not Diamond API request 39 | ========================================== 40 | 41 | Create a new file in the same directory as your `.env` file and copy and run the code below (you can toggle between Python and TypeScript in the top left of the code block): 42 | 43 | .. code:: python 44 | 45 | from notdiamond import NotDiamond 46 | 47 | # Define the Not Diamond routing client 48 | client = NotDiamond() 49 | 50 | # The best LLM is determined by Not Diamond based on the messages and specified models 51 | result, session_id, provider = client.chat.completions.create( 52 | messages=[ 53 | {"role": "system", "content": "You are a helpful assistant."}, 54 | {"role": "user", "content": "Consiely explain merge sort."} # Adjust as desired 55 | ], 56 | model=['openai/gpt-3.5-turbo', 'openai/gpt-4o', 'anthropic/claude-3-5-sonnet-20240620'] 57 | ) 58 | 59 | print("ND session ID: ", session_id) # A unique ID of Not Diamond's recommendation 60 | print("LLM called: ", provider.model) # The LLM routed to 61 | print("LLM output: ", result.content) # The LLM response 62 | -------------------------------------------------------------------------------- /docs/source/notdiamond.llms.rst: -------------------------------------------------------------------------------- 1 | notdiamond.llms 2 | ======================= 3 | 4 | 5 | notdiamond.llms.client 6 | -------------------------- 7 | 8 | .. autoclass:: notdiamond.llms.client.NotDiamond 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | notdiamond.llms.config 14 | ------------------------------- 15 | 16 | .. automodule:: notdiamond.llms.config 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | 21 | notdiamond.llms.providers 22 | -------------------------------- 23 | 24 | .. automodule:: notdiamond.llms.providers 25 | :members: 26 | :undoc-members: 27 | :show-inheritance: 28 | :no-index: 29 | 30 | notdiamond.llms.request 31 | ------------------------------ 32 | 33 | .. automodule:: notdiamond.llms.request 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | -------------------------------------------------------------------------------- /docs/source/notdiamond.metrics.rst: -------------------------------------------------------------------------------- 1 | notdiamond.metrics 2 | ================== 3 | 4 | 5 | notdiamond.metrics.metric 6 | ------------------------- 7 | 8 | .. automodule:: notdiamond.metrics.metric 9 | :members: 10 | :undoc-members: 11 | :show-inheritance: 12 | 13 | notdiamond.metrics.request 14 | -------------------------- 15 | 16 | .. automodule:: notdiamond.metrics.request 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | -------------------------------------------------------------------------------- /docs/source/notdiamond.rst: -------------------------------------------------------------------------------- 1 | NotDiamond package 2 | ================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | notdiamond.llms 11 | notdiamond.metrics 12 | notdiamond.toolkit 13 | 14 | Submodules 15 | ---------- 16 | 17 | .. toctree:: 18 | :maxdepth: 4 19 | 20 | notdiamond._utils 21 | notdiamond.callbacks 22 | notdiamond.exceptions 23 | notdiamond.prompts 24 | notdiamond.types 25 | 26 | Module contents 27 | -------------- 28 | 29 | .. automodule:: notdiamond 30 | :members: 31 | :undoc-members: 32 | :show-inheritance: 33 | 34 | .. autofunction:: notdiamond.init 35 | -------------------------------------------------------------------------------- /docs/source/notdiamond.toolkit.rag.rst: -------------------------------------------------------------------------------- 1 | notdiamond.toolkit.rag 2 | ====================== 3 | 4 | notdiamond.toolkit.rag.document_loaders 5 | --------------------------------------- 6 | 7 | .. automodule:: notdiamond.toolkit.rag.document_loaders 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | 12 | notdiamond.toolkit.rag.evaluation_dataset 13 | ----------------------------------------- 14 | 15 | .. automodule:: notdiamond.toolkit.rag.evaluation_dataset 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | :special-members: __init__ 20 | :exclude-members: from_list, generation_prompt, generator_llm 21 | 22 | notdiamond.toolkit.rag.evaluation 23 | ----------------------------------------- 24 | 25 | .. automodule:: notdiamond.toolkit.rag.evaluation 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | 30 | notdiamond.toolkit.rag.llms 31 | ------------------------------- 32 | 33 | .. automodule:: notdiamond.toolkit.rag.llms 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | 38 | notdiamond.toolkit.rag.metrics 39 | ------------------------------- 40 | 41 | .. automodule:: notdiamond.toolkit.rag.metrics 42 | :members: 43 | :undoc-members: 44 | :show-inheritance: 45 | 46 | notdiamond.toolkit.rag.testset 47 | ------------------------------- 48 | 49 | .. automodule:: notdiamond.toolkit.rag.testset 50 | :members: 51 | :undoc-members: 52 | :show-inheritance: 53 | 54 | notdiamond.toolkit.rag.workflow 55 | ------------------------------- 56 | 57 | .. automodule:: notdiamond.toolkit.rag.workflow 58 | :members: 59 | :undoc-members: 60 | :show-inheritance: 61 | -------------------------------------------------------------------------------- /docs/source/notdiamond.toolkit.rst: -------------------------------------------------------------------------------- 1 | notdiamond.toolkit package 2 | ====================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | .. toctree:: 8 | :maxdepth: 4 9 | 10 | notdiamond.toolkit.rag 11 | 12 | Module contents 13 | -------------- 14 | 15 | .. automodule:: notdiamond.toolkit 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | 20 | notdiamond.toolkit.custom_router 21 | -------------------------------- 22 | 23 | .. autoclass:: notdiamond.toolkit.custom_router.CustomRouter 24 | :members: 25 | :undoc-members: 26 | :show-inheritance: 27 | -------------------------------------------------------------------------------- /notdiamond/__init__.py: -------------------------------------------------------------------------------- 1 | from notdiamond.llms.client import NotDiamond # noqa: F401 2 | from notdiamond.llms.config import LLMConfig # noqa: F401 3 | from notdiamond.metrics.metric import Metric # noqa: F401 4 | 5 | __all__ = [] 6 | try: 7 | import importlib.util 8 | 9 | spec = importlib.util.find_spec("openai") 10 | if spec is not None: 11 | from ._init import init # noqa: F401 12 | 13 | __all__ = ["init"] 14 | else: 15 | __all__ = [] 16 | except ImportError: 17 | pass 18 | -------------------------------------------------------------------------------- /notdiamond/callbacks.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Any 3 | 4 | from notdiamond._utils import _module_check 5 | from notdiamond.llms.config import LLMConfig 6 | 7 | LOGGER = logging.getLogger(__name__) 8 | LOGGER.setLevel(logging.INFO) 9 | 10 | 11 | def _nd_callback_handler_factory(): 12 | try: 13 | BaseCallbackHandler = _module_check( 14 | "langchain_core.callbacks.base", "BaseCallbackHandler" 15 | ) 16 | except (ModuleNotFoundError, ImportError) as ierr: 17 | log_msg = ( 18 | "Attempted to instantiate NDLLMBaseCallbackHandler but langchain is not installed. ", 19 | f"Model callbacks not available. {ierr}", 20 | ) 21 | LOGGER.warning(log_msg) 22 | return 23 | 24 | class _NDLLMBaseCallbackHandler(BaseCallbackHandler): 25 | """ 26 | Base callback handler for NotDiamond LLMs. 27 | Accepts all of the langchain_core callbacks and adds new ones. 28 | """ 29 | 30 | def on_model_select( 31 | self, model_provider: LLMConfig, model_name: str 32 | ) -> Any: 33 | """ 34 | Called when a model is selected. 35 | """ 36 | 37 | def on_latency_tracking( 38 | self, 39 | session_id: str, 40 | model_provider: LLMConfig, 41 | tokens_per_second: float, 42 | ): 43 | """ 44 | Called when latency tracking is enabled. 45 | """ 46 | 47 | def on_api_error(self, error_message: str): 48 | """ 49 | Called when an NotDiamond API error occurs. 50 | """ 51 | 52 | return _NDLLMBaseCallbackHandler 53 | 54 | 55 | NDLLMBaseCallbackHandler = _nd_callback_handler_factory() 56 | -------------------------------------------------------------------------------- /notdiamond/exceptions.py: -------------------------------------------------------------------------------- 1 | class UnsupportedLLMProvider(Exception): 2 | """The exception class for unsupported LLM provider""" 3 | 4 | 5 | class UnsupportedEmbeddingProvider(Exception): 6 | """The exception class for unsupported Embedding provider""" 7 | 8 | 9 | class InvalidApiKey(Exception): 10 | """The exception class for InvalidApiKey""" 11 | 12 | 13 | class MissingApiKey(Exception): 14 | """The exception class for MissingApiKey""" 15 | 16 | 17 | class MissingLLMConfigs(Exception): 18 | """The exception class for empty LLM provider configs array""" 19 | 20 | 21 | class ApiError(Exception): 22 | """The exception class for any ApiError""" 23 | 24 | 25 | class CreateUnavailableError(Exception): 26 | """The exception class raised when `notdiamond[create]` is not available""" 27 | -------------------------------------------------------------------------------- /notdiamond/llms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/notdiamond/llms/__init__.py -------------------------------------------------------------------------------- /notdiamond/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/notdiamond/metrics/__init__.py -------------------------------------------------------------------------------- /notdiamond/metrics/metric.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from notdiamond import settings 4 | from notdiamond.exceptions import ApiError 5 | from notdiamond.llms.config import LLMConfig 6 | from notdiamond.metrics.request import feedback_request 7 | from notdiamond.types import NDApiKeyValidator 8 | 9 | 10 | class Metric: 11 | def __init__(self, metric: Optional[str] = "accuracy"): 12 | self.metric = metric 13 | 14 | def __call__(self): 15 | return self.metric 16 | 17 | def feedback( 18 | self, 19 | session_id: str, 20 | llm_config: LLMConfig, 21 | value: int, 22 | notdiamond_api_key: Optional[str] = None, 23 | _user_agent: str = None, 24 | ): 25 | if notdiamond_api_key is None: 26 | notdiamond_api_key = settings.NOTDIAMOND_API_KEY 27 | NDApiKeyValidator(api_key=notdiamond_api_key) 28 | if value not in [0, 1]: 29 | raise ApiError("Invalid feedback value. It must be 0 or 1.") 30 | 31 | return feedback_request( 32 | session_id=session_id, 33 | llm_config=llm_config, 34 | feedback_payload=self.request_payload(value), 35 | notdiamond_api_key=notdiamond_api_key, 36 | _user_agent=_user_agent, 37 | ) 38 | 39 | def request_payload(self, value: int): 40 | return {self.metric: value} 41 | -------------------------------------------------------------------------------- /notdiamond/metrics/request.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Dict 3 | 4 | import requests 5 | 6 | from notdiamond import settings 7 | from notdiamond._utils import _default_headers 8 | from notdiamond.exceptions import ApiError 9 | from notdiamond.llms.config import LLMConfig 10 | from notdiamond.types import FeedbackRequestPayload 11 | 12 | LOGGER = logging.getLogger(__name__) 13 | LOGGER.setLevel(logging.INFO) 14 | 15 | 16 | def feedback_request( 17 | session_id: str, 18 | llm_config: LLMConfig, 19 | feedback_payload: Dict[str, int], 20 | notdiamond_api_key: str, 21 | nd_api_url: str = settings.NOTDIAMOND_API_URL, 22 | _user_agent: str = settings.DEFAULT_USER_AGENT, 23 | ) -> bool: 24 | url = f"{nd_api_url}/v2/report/metrics/feedback" 25 | 26 | payload: FeedbackRequestPayload = { 27 | "session_id": session_id, 28 | "provider": llm_config.prepare_for_request(), 29 | "feedback": feedback_payload, 30 | } 31 | 32 | headers = _default_headers(notdiamond_api_key, _user_agent) 33 | 34 | try: 35 | response = requests.post(url, json=payload, headers=headers) 36 | except Exception as e: 37 | raise ApiError(f"ND API error for feedback: {e}") 38 | 39 | if response.status_code != 200: 40 | LOGGER.error( 41 | f"ND API feedback error: failed to report feedback with status {response.status_code}. {response.text}" 42 | ) 43 | return False 44 | 45 | return True 46 | -------------------------------------------------------------------------------- /notdiamond/prompts.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | from typing import Dict, List 4 | 5 | from notdiamond.llms.config import LLMConfig 6 | from notdiamond.llms.providers import is_o1_model 7 | 8 | LOGGER = logging.getLogger(__name__) 9 | LOGGER.setLevel(logging.INFO) 10 | 11 | 12 | def inject_system_prompt( 13 | messages: List[Dict[str, str]], system_prompt: str 14 | ) -> List[Dict[str, str]]: 15 | """ 16 | Add a system prompt to an OpenAI-style message list. If a system prompt is already present, replace it. 17 | """ 18 | new_messages = [] 19 | found = False 20 | for msg in messages: 21 | # t7: replace the first system prompt with the new one 22 | if msg["role"] == "system" and not found: 23 | new_messages.append({"role": "system", "content": system_prompt}) 24 | found = True 25 | else: 26 | new_messages.append(msg) 27 | if not found: 28 | new_messages.insert(0, {"role": "system", "content": system_prompt}) 29 | return new_messages 30 | 31 | 32 | def _curly_escape(text: str) -> str: 33 | """ 34 | Escape curly braces in the text, but only for single occurrences of alphabetic characters. 35 | This function will not escape double curly braces or non-alphabetic characters. 36 | """ 37 | return re.sub(r"(? List[Dict[str, str]]: 43 | if is_o1_model(llm): 44 | translated_messages = [] 45 | for msg in messages: 46 | if msg["role"] == "system": 47 | translated_messages.append( 48 | {"role": "user", "content": msg["content"]} 49 | ) 50 | else: 51 | translated_messages.append(msg) 52 | return translated_messages 53 | return messages 54 | -------------------------------------------------------------------------------- /notdiamond/toolkit/__init__.py: -------------------------------------------------------------------------------- 1 | from .custom_router import CustomRouter # noqa 2 | -------------------------------------------------------------------------------- /notdiamond/toolkit/rag/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Not-Diamond/notdiamond-python/4fc1e086925864cb0988b3afaa7c543035582960/notdiamond/toolkit/rag/__init__.py -------------------------------------------------------------------------------- /notdiamond/toolkit/rag/document_loaders.py: -------------------------------------------------------------------------------- 1 | from langchain_community.document_loaders import DirectoryLoader # noqa 2 | -------------------------------------------------------------------------------- /notdiamond/toolkit/rag/evaluation_dataset.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Dict, List, Union, overload 3 | 4 | from ragas import MultiTurnSample, SingleTurnSample 5 | from ragas.dataset_schema import RagasDataset 6 | 7 | 8 | class RAGSample(SingleTurnSample): 9 | """ 10 | Represents RAG evaluation samples. 11 | 12 | Attributes: 13 | user_input (Optional[str]): The input query from the user. 14 | retrieved_contexts (Optional[List[str]]): List of contexts retrieved for the query. 15 | reference_contexts (Optional[List[str]]): List of reference contexts for the query. 16 | response (Optional[str]): The generated response for the query. 17 | generation_prompt (str): The input prompt to the generator LLM. 18 | generator_llm (str): The LLM used to generate the response. 19 | multi_responses (Optional[List[str]]): List of multiple responses generated for the query. 20 | reference (Optional[str]): The reference answer for the query. 21 | rubric (Optional[Dict[str, str]]): Evaluation rubric for the sample. 22 | """ 23 | 24 | generation_prompt: str 25 | generator_llm: str 26 | 27 | 28 | @dataclass 29 | class RAGEvaluationDataset(RagasDataset[RAGSample]): 30 | """ 31 | Represents a dataset of RAG evaluation samples. 32 | 33 | Attributes: 34 | samples (List[BaseSample]): A list of evaluation samples. 35 | 36 | Methods: 37 | validate_samples(samples): Validates that all samples are of the same type. 38 | get_sample_type(): Returns the type of the samples in the dataset. 39 | to_hf_dataset(): Converts the dataset to a Hugging Face Dataset. 40 | to_pandas(): Converts the dataset to a pandas DataFrame. 41 | features(): Returns the features of the samples. 42 | from_list(mapping): Creates an EvaluationDataset from a list of dictionaries. 43 | from_dict(mapping): Creates an EvaluationDataset from a dictionary. 44 | to_csv(path): Converts the dataset to a CSV file. 45 | to_jsonl(path): Converts the dataset to a JSONL file. 46 | from_jsonl(path): Creates an EvaluationDataset from a JSONL file. 47 | """ 48 | 49 | @overload 50 | def __getitem__(self, idx: int) -> RAGSample: 51 | ... 52 | 53 | @overload 54 | def __getitem__(self, idx: slice) -> "RAGEvaluationDataset": 55 | ... 56 | 57 | def __getitem__( 58 | self, idx: Union[int, slice] 59 | ) -> Union[RAGSample, "RAGEvaluationDataset"]: 60 | if isinstance(idx, int): 61 | return self.samples[idx] 62 | elif isinstance(idx, slice): 63 | return type(self)(samples=self.samples[idx]) 64 | else: 65 | raise TypeError("Index must be int or slice") 66 | 67 | def is_multi_turn(self) -> bool: 68 | return False 69 | 70 | def to_list(self) -> List[Dict]: 71 | rows = [sample.to_dict() for sample in self.samples] 72 | return rows 73 | 74 | @classmethod 75 | def from_list(cls, data: List[Dict]): 76 | samples = [] 77 | if all( 78 | "user_input" in item and isinstance(data[0]["user_input"], list) 79 | for item in data 80 | ): 81 | samples.extend(MultiTurnSample(**sample) for sample in data) 82 | else: 83 | samples.extend(SingleTurnSample(**sample) for sample in data) 84 | return cls(samples=samples) 85 | 86 | def __repr__(self) -> str: 87 | return f"RAGEvaluationDataset(features={self.features()}, len={len(self.samples)})" 88 | -------------------------------------------------------------------------------- /notdiamond/toolkit/rag/llms.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | from langchain_cohere import CohereEmbeddings 4 | from langchain_mistralai import MistralAIEmbeddings 5 | from langchain_openai import OpenAIEmbeddings 6 | from ragas.embeddings import HuggingfaceEmbeddings, LangchainEmbeddingsWrapper 7 | from ragas.llms import LangchainLLMWrapper 8 | 9 | from ...exceptions import UnsupportedEmbeddingProvider 10 | from ...llms.client import NotDiamond 11 | from ...llms.config import EmbeddingConfig, LLMConfig 12 | 13 | 14 | def get_llm(llm_config_or_str: Union[LLMConfig, str]) -> LangchainLLMWrapper: 15 | """ 16 | Build the LLM object compatible with evaluation metrics. 17 | 18 | Parameters: 19 | llm_config_or_str (Union[LLMConfig, str]): a LLMConfig object or a model string 20 | that specifies the LLM to construct. 21 | """ 22 | if isinstance(llm_config_or_str, str): 23 | llm_config = LLMConfig.from_string(llm_config_or_str) 24 | else: 25 | llm_config = llm_config_or_str 26 | 27 | lc_llm = NotDiamond._llm_from_config(llm_config) 28 | return LangchainLLMWrapper(lc_llm) 29 | 30 | 31 | def get_embedding( 32 | embedding_model_config_or_str: Union[EmbeddingConfig, str] 33 | ) -> Union[LangchainEmbeddingsWrapper, HuggingfaceEmbeddings]: 34 | """ 35 | Build the embedding model object compatible with evaluation metrics. 36 | 37 | Parameters: 38 | embedding_model_config_or_str (Union[EmbeddingConfig, str]): an EmbeddingConfig object 39 | or an embedding model string that specifies the embedding model to construct. 40 | """ 41 | if isinstance(embedding_model_config_or_str, str): 42 | embedding_config = EmbeddingConfig.from_string( 43 | embedding_model_config_or_str 44 | ) 45 | else: 46 | embedding_config = embedding_model_config_or_str 47 | 48 | if embedding_config.provider == "openai": 49 | lc_embedding = OpenAIEmbeddings( 50 | model=embedding_config.model, **embedding_config.kwargs 51 | ) 52 | 53 | elif embedding_config.provider == "cohere": 54 | lc_embedding = CohereEmbeddings( 55 | model=embedding_config.model, **embedding_config.kwargs 56 | ) 57 | 58 | elif embedding_config.provider == "mistral": 59 | lc_embedding = MistralAIEmbeddings( 60 | model=embedding_config.model, **embedding_config.kwargs 61 | ) 62 | 63 | elif embedding_config.provider == "huggingface": 64 | return HuggingfaceEmbeddings(model_name=embedding_config.model) 65 | 66 | else: 67 | raise UnsupportedEmbeddingProvider( 68 | f"Embedding model {str(embedding_config)} not supported." 69 | ) 70 | return LangchainEmbeddingsWrapper(lc_embedding) 71 | -------------------------------------------------------------------------------- /notdiamond/toolkit/rag/metrics.py: -------------------------------------------------------------------------------- 1 | from ragas.metrics import ( # noqa 2 | AspectCritic, 3 | BleuScore, 4 | ContextEntityRecall, 5 | ContextPrecision, 6 | ContextRecall, 7 | ExactMatch, 8 | FactualCorrectness, 9 | Faithfulness, 10 | FaithfulnesswithHHEM, 11 | InstanceRubrics, 12 | ) 13 | from ragas.metrics import ( # noqa 14 | LLMContextPrecisionWithoutReference as ContextPrecisionWithoutReference, 15 | ) 16 | from ragas.metrics import ( # noqa 17 | LLMContextPrecisionWithReference, 18 | LLMContextRecall, 19 | NoiseSensitivity, 20 | NonLLMContextPrecisionWithReference, 21 | NonLLMContextRecall, 22 | NonLLMStringSimilarity, 23 | ResponseRelevancy, 24 | RougeScore, 25 | RubricsScore, 26 | SemanticSimilarity, 27 | SimpleCriteriaScore, 28 | StringPresence, 29 | ) 30 | -------------------------------------------------------------------------------- /notdiamond/types.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, List 2 | 3 | from pydantic import BaseModel, field_validator 4 | 5 | from notdiamond.exceptions import InvalidApiKey, MissingApiKey 6 | 7 | 8 | class NDApiKeyValidator(BaseModel): 9 | api_key: str 10 | 11 | @field_validator("api_key", mode="before") 12 | @classmethod 13 | def api_key_must_be_a_string(cls, v) -> str: 14 | if not isinstance(v, str): 15 | raise InvalidApiKey("ND API key should be a string") 16 | return v 17 | 18 | @field_validator("api_key", mode="after") 19 | @classmethod 20 | def string_must_not_be_empty(cls, v): 21 | if len(v) == 0: 22 | raise MissingApiKey("ND API key should be longer than 0") 23 | return v 24 | 25 | 26 | class ModelSelectRequestPayload(BaseModel): 27 | prompt_template: str 28 | formatted_prompt: str 29 | components: Dict[str, Dict] 30 | llm_configs: List[Dict] 31 | metric: str 32 | max_model_depth: int 33 | 34 | 35 | class FeedbackRequestPayload(BaseModel): 36 | session_id: str 37 | provider: Dict[str, Any] 38 | feedback: Dict[str, int] 39 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "notdiamond" 3 | version = "0.4.4" 4 | description = "Not Diamond Python SDK" 5 | authors = ["Not Diamond "] 6 | readme = "README.md" 7 | homepage = "https://python.notdiamond.ai/" 8 | repository = "https://github.com/Not-Diamond/notdiamond-python" 9 | documentation = "https://notdiamond.readme.io/docs/" 10 | 11 | [tool.poetry.dependencies] 12 | python = ">3.10,<3.13" 13 | python-dotenv = "^1.0.1" 14 | aiohttp = "^3.9.3" 15 | pandas = "^2.2.2" 16 | pydantic-partial = "^0.5.5" 17 | cryptography = ">=44.0.1" 18 | h11 = ">=0.16.0" 19 | tiktoken = ">=0.8.0,<0.9.0" 20 | langchain = { version = ">=0.3.22", optional = true } 21 | langchain-google-genai = { version = ">=2.1.2", optional = true } 22 | langchain-openai = { version = ">=0.3.9", optional = true } 23 | langchain-anthropic = { version = ">=0.3.10", optional = true } 24 | langchain-community = { version = ">=0.3.20", optional = true } 25 | langchain-mistralai = { version = ">=0.2.10", optional = true } 26 | langchain-together = { version = ">=0.3.0", optional = true } 27 | langchain-cohere = { version = ">=0.4.3", optional = true } 28 | requests = ">=2.31" 29 | optuna = { version = "^4.1.0", optional = true } 30 | ragas = { version = "^0.2.6", optional = true } 31 | unstructured = { extras = [ 32 | "local-inference", 33 | ], version = "^0.16.5", optional = true } 34 | onnxruntime = { version = "<1.20", optional = true } 35 | python-multipart = { version = "0.0.18", extras = ["rag"] } 36 | llama-index = { version = "^0.12.25", optional = true } 37 | torch = { version = "^2.7.0", optional = true } 38 | 39 | [tool.poetry.group.test.dependencies] 40 | pytest = "^8.0.0" 41 | pytest-asyncio = "^0.23.6" 42 | pytest-mock = "^3.14.0" 43 | openai = ">=1.66.3" 44 | coverage = "^7.6.0" 45 | pytest-recording = "^0.13.2" 46 | pytest-timeout = "^2.3.1" 47 | 48 | [tool.poetry.group.dev.dependencies] 49 | ruff = "^0.3.1" 50 | pydantic = "^2.5.3" 51 | pre-commit = "^3.7.0" 52 | black = "^24.3.0" 53 | flake8 = "^7.0.0" 54 | poetry-pre-commit-plugin = "^0.1.2" 55 | isort = "^5.13.2" 56 | nox = "^2024.3.2" 57 | pytest-mock = "^3.14.0" 58 | 59 | 60 | [tool.poetry.group.docs.dependencies] 61 | sphinx = "^7.3.7" 62 | autodoc = "^0.5.0" 63 | sphinx-rtd-theme = "^2.0.0" 64 | toml = "^0.10.2" 65 | 66 | [tool.poetry.extras] 67 | create = [ 68 | "langchain-cohere", 69 | "langchain-anthropic", 70 | "langchain-community", 71 | "langchain-mistralai", 72 | "langchain-google-genai", 73 | "langchain-openai", 74 | "langchain-together", 75 | "langchain", 76 | ] 77 | openai = ["openai"] 78 | rag = [ 79 | "ragas", 80 | "unstructured", 81 | "optuna", 82 | "onnxruntime", 83 | "langchain-cohere", 84 | "langchain-anthropic", 85 | "langchain-community", 86 | "langchain-mistralai", 87 | "langchain-google-genai", 88 | "langchain-openai", 89 | "langchain-together", 90 | "langchain", 91 | "llama-index", 92 | "torch", 93 | ] 94 | 95 | [build-system] 96 | requires = ["poetry-core"] 97 | build-backend = "poetry.core.masonry.api" 98 | 99 | [tool.pytest.ini_options] 100 | markers = ["longrun"] 101 | 102 | [tool.black] 103 | line-length = 79 104 | target-version = ['py39'] 105 | include = '\.pyi?$' 106 | exclude = ''' 107 | /( 108 | \.git 109 | | \.mypy_cache 110 | | \.venv 111 | | _build 112 | )/ 113 | ''' 114 | 115 | [tool.flake8] 116 | max-line-length = 130 117 | extend-ignore = [ 118 | "D203", 119 | "E203", 120 | "E251", 121 | "E266", 122 | "E302", 123 | "E305", 124 | "E401", 125 | "E402", 126 | "E501", 127 | "F401", 128 | "F403", 129 | "W503", 130 | ] 131 | exclude = [".git", "__pycache__", "dist", "cookbooks"] 132 | max-complexity = 10 133 | 134 | [tool.isort] 135 | atomic = true 136 | profile = "black" 137 | line_length = 79 138 | skip_gitignore = true 139 | 140 | [tool.coverage.run] 141 | branch = true 142 | relative_files = true 143 | 144 | [tool.coverage.report] 145 | exclude_also = [ 146 | # Don't complain about abstract methods, they aren't run: 147 | "@(abc\\.)?abstractmethod", 148 | ] 149 | -------------------------------------------------------------------------------- /tests/helpers.py: -------------------------------------------------------------------------------- 1 | def stream_chunks(ichunk): 2 | n_chunks = 0 3 | all_empty = True 4 | for chunk in ichunk: 5 | assert chunk.type == "AIMessageChunk" 6 | assert isinstance(chunk.content, str) 7 | all_empty &= len(chunk.content) == 0 8 | n_chunks += 1 9 | if n_chunks >= 5: 10 | break 11 | assert not all_empty 12 | 13 | 14 | async def astream_chunks(async_ichunk): 15 | n_chunks = 0 16 | all_empty = True 17 | async for chunk in async_ichunk: 18 | assert chunk.type == "AIMessageChunk" 19 | assert isinstance(chunk.content, str) 20 | all_empty &= len(chunk.content) == 0 21 | n_chunks += 1 22 | if n_chunks >= 5: 23 | break 24 | assert not all_empty 25 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_callbacks/test_model_select_callback[_NDInvokerClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Hello, what''s your name?"}], 4 | "llm_providers": [{"provider": "openai", "model": "gpt-4-turbo-preview", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}, {"provider": "openai", "model": "gpt-3.5-turbo", "is_custom": false, 7 | "context_length": null, "input_price": null, "output_price": null, "latency": 8 | null}], "metric": "accuracy", "max_model_depth": 2, "hash_content": false}' 9 | headers: 10 | Accept: 11 | - '*/*' 12 | Accept-Encoding: 13 | - gzip, deflate, zstd 14 | Connection: 15 | - keep-alive 16 | Content-Length: 17 | - '471' 18 | User-Agent: 19 | - Python-SDK/0.3.41 20 | content-type: 21 | - application/json 22 | method: POST 23 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 24 | response: 25 | body: 26 | string: '{"providers":[{"provider":"openai","model":"gpt-3.5-turbo"}],"session_id":"89489a7c-694f-46da-91a2-c7a763837d8e"}' 27 | headers: 28 | CF-RAY: 29 | - 925cfab0cc25c228-VIE 30 | Connection: 31 | - keep-alive 32 | Content-Type: 33 | - application/json 34 | Date: 35 | - Tue, 25 Mar 2025 08:18:44 GMT 36 | Server: 37 | - cloudflare 38 | Transfer-Encoding: 39 | - chunked 40 | alt-svc: 41 | - h3=":443"; ma=86400 42 | cf-cache-status: 43 | - DYNAMIC 44 | content-length: 45 | - '113' 46 | rndr-id: 47 | - 9709566f-eeb7-4edb 48 | vary: 49 | - Accept-Encoding 50 | x-render-origin-server: 51 | - uvicorn 52 | status: 53 | code: 200 54 | message: OK 55 | version: 1 56 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_callbacks/test_model_select_callback[_NDRouterClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Hello, what''s your name?"}], 4 | "llm_providers": [{"provider": "openai", "model": "gpt-4-turbo-preview", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}, {"provider": "openai", "model": "gpt-3.5-turbo", "is_custom": false, 7 | "context_length": null, "input_price": null, "output_price": null, "latency": 8 | null}], "metric": "accuracy", "max_model_depth": 2, "hash_content": false}' 9 | headers: 10 | Accept: 11 | - '*/*' 12 | Accept-Encoding: 13 | - gzip, deflate, zstd 14 | Connection: 15 | - keep-alive 16 | Content-Length: 17 | - '471' 18 | User-Agent: 19 | - Python-SDK/0.3.41 20 | content-type: 21 | - application/json 22 | method: POST 23 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 24 | response: 25 | body: 26 | string: '{"providers":[{"provider":"openai","model":"gpt-3.5-turbo"}],"session_id":"35b450dc-40d8-4559-b0c2-2c9adbee9356"}' 27 | headers: 28 | CF-RAY: 29 | - 925cfaac2cc1c268-VIE 30 | Connection: 31 | - keep-alive 32 | Content-Type: 33 | - application/json 34 | Date: 35 | - Tue, 25 Mar 2025 08:18:43 GMT 36 | Server: 37 | - cloudflare 38 | Transfer-Encoding: 39 | - chunked 40 | alt-svc: 41 | - h3=":443"; ma=86400 42 | cf-cache-status: 43 | - DYNAMIC 44 | content-length: 45 | - '113' 46 | rndr-id: 47 | - c59c84ee-6b17-458b 48 | vary: 49 | - Accept-Encoding 50 | x-render-origin-server: 51 | - uvicorn 52 | status: 53 | code: 200 54 | message: OK 55 | version: 1 56 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_async_model_select_with_strings[_NDClientTarget.INVOKER-_NDInvokerClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Write me a song about goldfish 4 | on the moon"}], "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo", 5 | "is_custom": false, "context_length": null, "input_price": null, "output_price": 6 | null, "latency": null}, {"provider": "openai", "model": "gpt-4", "is_custom": 7 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 8 | null}, {"provider": "anthropic", "model": "claude-2.1", "is_custom": false, 9 | "context_length": null, "input_price": null, "output_price": null, "latency": 10 | null}, {"provider": "google", "model": "gemini-pro", "is_custom": false, "context_length": 11 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 12 | "accuracy", "max_model_depth": 4, "hash_content": true}' 13 | headers: 14 | User-Agent: 15 | - Python-SDK/0.3.41 16 | content-type: 17 | - application/json 18 | method: POST 19 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 20 | response: 21 | body: 22 | string: '{"providers":[{"provider":"openai","model":"gpt-4"}],"session_id":"7a3032f4-cf00-45d3-9f45-f1f61830abbf"}' 23 | headers: 24 | CF-RAY: 25 | - 925cfae498ddc747-VIE 26 | Connection: 27 | - keep-alive 28 | Content-Type: 29 | - application/json 30 | Date: 31 | - Tue, 25 Mar 2025 08:18:52 GMT 32 | Server: 33 | - cloudflare 34 | Transfer-Encoding: 35 | - chunked 36 | Vary: 37 | - Accept-Encoding 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | content-length: 43 | - '105' 44 | rndr-id: 45 | - cd53e938-6152-4e22 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 200 50 | message: OK 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_async_model_select_with_strings[_NDClientTarget.ROUTER-_NDRouterClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Write me a song about goldfish 4 | on the moon"}], "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo", 5 | "is_custom": false, "context_length": null, "input_price": null, "output_price": 6 | null, "latency": null}, {"provider": "openai", "model": "gpt-4", "is_custom": 7 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 8 | null}, {"provider": "anthropic", "model": "claude-2.1", "is_custom": false, 9 | "context_length": null, "input_price": null, "output_price": null, "latency": 10 | null}, {"provider": "google", "model": "gemini-pro", "is_custom": false, "context_length": 11 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 12 | "accuracy", "max_model_depth": 4, "hash_content": true}' 13 | headers: 14 | User-Agent: 15 | - Python-SDK/0.3.41 16 | content-type: 17 | - application/json 18 | method: POST 19 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 20 | response: 21 | body: 22 | string: '{"providers":[{"provider":"openai","model":"gpt-4"}],"session_id":"3625a798-925a-4435-9855-1ab93d0ed74f"}' 23 | headers: 24 | CF-RAY: 25 | - 925cfadf29645b4e-VIE 26 | Connection: 27 | - keep-alive 28 | Content-Type: 29 | - application/json 30 | Date: 31 | - Tue, 25 Mar 2025 08:18:52 GMT 32 | Server: 33 | - cloudflare 34 | Transfer-Encoding: 35 | - chunked 36 | Vary: 37 | - Accept-Encoding 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | content-length: 43 | - '105' 44 | rndr-id: 45 | - 1e7b850b-bbb4-4ef2 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 200 50 | message: OK 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_model_select_with_strings[_NDClientTarget.INVOKER-_NDInvokerClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Write me a song about goldfish 4 | on the moon"}], "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo", 5 | "is_custom": false, "context_length": null, "input_price": null, "output_price": 6 | null, "latency": null}, {"provider": "openai", "model": "gpt-4", "is_custom": 7 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 8 | null}, {"provider": "anthropic", "model": "claude-2.1", "is_custom": false, 9 | "context_length": null, "input_price": null, "output_price": null, "latency": 10 | null}, {"provider": "google", "model": "gemini-pro", "is_custom": false, "context_length": 11 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 12 | "accuracy", "max_model_depth": 4, "hash_content": true}' 13 | headers: 14 | Accept: 15 | - '*/*' 16 | Accept-Encoding: 17 | - gzip, deflate, zstd 18 | Connection: 19 | - keep-alive 20 | Content-Length: 21 | - '779' 22 | User-Agent: 23 | - Python-SDK/0.3.41 24 | content-type: 25 | - application/json 26 | method: POST 27 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 28 | response: 29 | body: 30 | string: '{"providers":[{"provider":"openai","model":"gpt-4"}],"session_id":"5d496dbd-2fc8-4044-a018-fec52ad9f39c"}' 31 | headers: 32 | CF-RAY: 33 | - 925cfadabdd3c1f9-VIE 34 | Connection: 35 | - keep-alive 36 | Content-Type: 37 | - application/json 38 | Date: 39 | - Tue, 25 Mar 2025 08:18:51 GMT 40 | Server: 41 | - cloudflare 42 | Transfer-Encoding: 43 | - chunked 44 | alt-svc: 45 | - h3=":443"; ma=86400 46 | cf-cache-status: 47 | - DYNAMIC 48 | content-length: 49 | - '105' 50 | rndr-id: 51 | - 2d7f2dc4-0459-47a7 52 | vary: 53 | - Accept-Encoding 54 | x-render-origin-server: 55 | - uvicorn 56 | status: 57 | code: 200 58 | message: OK 59 | version: 1 60 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_model_select_with_strings[_NDClientTarget.ROUTER-_NDRouterClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Write me a song about goldfish 4 | on the moon"}], "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo", 5 | "is_custom": false, "context_length": null, "input_price": null, "output_price": 6 | null, "latency": null}, {"provider": "openai", "model": "gpt-4", "is_custom": 7 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 8 | null}, {"provider": "anthropic", "model": "claude-2.1", "is_custom": false, 9 | "context_length": null, "input_price": null, "output_price": null, "latency": 10 | null}, {"provider": "google", "model": "gemini-pro", "is_custom": false, "context_length": 11 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 12 | "accuracy", "max_model_depth": 4, "hash_content": true}' 13 | headers: 14 | Accept: 15 | - '*/*' 16 | Accept-Encoding: 17 | - gzip, deflate, zstd 18 | Connection: 19 | - keep-alive 20 | Content-Length: 21 | - '779' 22 | User-Agent: 23 | - Python-SDK/0.3.41 24 | content-type: 25 | - application/json 26 | method: POST 27 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 28 | response: 29 | body: 30 | string: '{"providers":[{"provider":"openai","model":"gpt-4"}],"session_id":"43eac436-56f1-4ec9-9dcd-4d51d1b31baf"}' 31 | headers: 32 | CF-RAY: 33 | - 925cfad5eebad100-SOF 34 | Connection: 35 | - keep-alive 36 | Content-Type: 37 | - application/json 38 | Date: 39 | - Tue, 25 Mar 2025 08:18:50 GMT 40 | Server: 41 | - cloudflare 42 | Transfer-Encoding: 43 | - chunked 44 | alt-svc: 45 | - h3=":443"; ma=86400 46 | cf-cache-status: 47 | - DYNAMIC 48 | content-length: 49 | - '105' 50 | rndr-id: 51 | - 5072108e-7aad-4c90 52 | vary: 53 | - Accept-Encoding 54 | x-render-origin-server: 55 | - uvicorn 56 | status: 57 | code: 200 58 | message: OK 59 | version: 1 60 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_preference_id[_NDClientTarget.INVOKER-_NDInvokerClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"name": "test"}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate, zstd 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '16' 13 | User-Agent: 14 | - Python-SDK/0.3.41 15 | content-type: 16 | - application/json 17 | method: POST 18 | uri: https://staging-api.notdiamond.ai/v2/preferences/userPreferenceCreate 19 | response: 20 | body: 21 | string: '{"preference_id":"bcf5f5c4-1a51-44b4-8054-efe8916e79c4"}' 22 | headers: 23 | CF-RAY: 24 | - 925cfba009f33db8-SOF 25 | Connection: 26 | - keep-alive 27 | Content-Type: 28 | - application/json 29 | Date: 30 | - Tue, 25 Mar 2025 08:19:22 GMT 31 | Server: 32 | - cloudflare 33 | Transfer-Encoding: 34 | - chunked 35 | alt-svc: 36 | - h3=":443"; ma=86400 37 | cf-cache-status: 38 | - DYNAMIC 39 | content-length: 40 | - '56' 41 | rndr-id: 42 | - 73c5294b-6a29-488c 43 | vary: 44 | - Accept-Encoding 45 | x-render-origin-server: 46 | - uvicorn 47 | status: 48 | code: 200 49 | message: OK 50 | version: 1 51 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_llm/Test_NDLLM.test_preference_id[_NDClientTarget.ROUTER-_NDRouterClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"name": "test"}' 4 | headers: 5 | Accept: 6 | - '*/*' 7 | Accept-Encoding: 8 | - gzip, deflate, zstd 9 | Connection: 10 | - keep-alive 11 | Content-Length: 12 | - '16' 13 | User-Agent: 14 | - Python-SDK/0.3.41 15 | content-type: 16 | - application/json 17 | method: POST 18 | uri: https://staging-api.notdiamond.ai/v2/preferences/userPreferenceCreate 19 | response: 20 | body: 21 | string: '{"preference_id":"defe1ede-025b-4e5d-b30e-3f0375fb9be8"}' 22 | headers: 23 | CF-RAY: 24 | - 925cfb9d6816d0da-SOF 25 | Connection: 26 | - keep-alive 27 | Content-Type: 28 | - application/json 29 | Date: 30 | - Tue, 25 Mar 2025 08:19:22 GMT 31 | Server: 32 | - cloudflare 33 | Transfer-Encoding: 34 | - chunked 35 | alt-svc: 36 | - h3=":443"; ma=86400 37 | cf-cache-status: 38 | - DYNAMIC 39 | content-length: 40 | - '56' 41 | rndr-id: 42 | - 02ea0db9-2544-4d9b 43 | vary: 44 | - Accept-Encoding 45 | x-render-origin-server: 46 | - uvicorn 47 | status: 48 | code: 200 49 | message: OK 50 | version: 1 51 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_llm/Test_OpenAI_style_input.test_openai_style_input_model_select[_NDClientTarget.INVOKER-_NDInvokerClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Hello, what''s your name?"}, 4 | {"role": "assistant", "content": "My name is Isaac Asimov"}, {"role": "user", 5 | "content": "And what do you do?"}], "llm_providers": [{"provider": "openai", 6 | "model": "gpt-3.5-turbo", "is_custom": false, "context_length": null, "input_price": 7 | null, "output_price": null, "latency": null}], "metric": "accuracy", "max_model_depth": 8 | 1, "hash_content": true}' 9 | headers: 10 | Accept: 11 | - '*/*' 12 | Accept-Encoding: 13 | - gzip, deflate, zstd 14 | Connection: 15 | - keep-alive 16 | Content-Length: 17 | - '423' 18 | User-Agent: 19 | - Python-SDK/0.3.41 20 | content-type: 21 | - application/json 22 | method: POST 23 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 24 | response: 25 | body: 26 | string: '{"providers":[{"provider":"openai","model":"gpt-3.5-turbo"}],"session_id":"38bb1226-b663-42e7-bbdd-dbeaec1f2604"}' 27 | headers: 28 | CF-RAY: 29 | - 925cfbd2f8f9c270-VIE 30 | Connection: 31 | - keep-alive 32 | Content-Type: 33 | - application/json 34 | Date: 35 | - Tue, 25 Mar 2025 08:19:31 GMT 36 | Server: 37 | - cloudflare 38 | Transfer-Encoding: 39 | - chunked 40 | alt-svc: 41 | - h3=":443"; ma=86400 42 | cf-cache-status: 43 | - DYNAMIC 44 | content-length: 45 | - '113' 46 | rndr-id: 47 | - ded4ecf8-c43a-4d5c 48 | vary: 49 | - Accept-Encoding 50 | x-render-origin-server: 51 | - uvicorn 52 | status: 53 | code: 200 54 | message: OK 55 | version: 1 56 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/cassettes/test_llm/Test_OpenAI_style_input.test_openai_style_input_model_select[_NDClientTarget.ROUTER-_NDRouterClient].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Hello, what''s your name?"}, 4 | {"role": "assistant", "content": "My name is Isaac Asimov"}, {"role": "user", 5 | "content": "And what do you do?"}], "llm_providers": [{"provider": "openai", 6 | "model": "gpt-3.5-turbo", "is_custom": false, "context_length": null, "input_price": 7 | null, "output_price": null, "latency": null}], "metric": "accuracy", "max_model_depth": 8 | 1, "hash_content": true}' 9 | headers: 10 | Accept: 11 | - '*/*' 12 | Accept-Encoding: 13 | - gzip, deflate, zstd 14 | Connection: 15 | - keep-alive 16 | Content-Length: 17 | - '423' 18 | User-Agent: 19 | - Python-SDK/0.3.41 20 | content-type: 21 | - application/json 22 | method: POST 23 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 24 | response: 25 | body: 26 | string: '{"providers":[{"provider":"openai","model":"gpt-3.5-turbo"}],"session_id":"858a6f9d-bd1d-4a0b-9991-fab4c05bb797"}' 27 | headers: 28 | CF-RAY: 29 | - 925cfbcec8a75b13-VIE 30 | Connection: 31 | - keep-alive 32 | Content-Type: 33 | - application/json 34 | Date: 35 | - Tue, 25 Mar 2025 08:19:30 GMT 36 | Server: 37 | - cloudflare 38 | Transfer-Encoding: 39 | - chunked 40 | alt-svc: 41 | - h3=":443"; ma=86400 42 | cf-cache-status: 43 | - DYNAMIC 44 | content-length: 45 | - '113' 46 | rndr-id: 47 | - 975085bd-655f-4f7c 48 | vary: 49 | - Accept-Encoding 50 | x-render-origin-server: 51 | - uvicorn 52 | status: 53 | code: 200 54 | message: OK 55 | version: 1 56 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/test_callbacks.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from notdiamond.llms.client import _NDClientTarget, _ndllm_factory 4 | from notdiamond.llms.providers import NDLLMProviders 5 | 6 | 7 | @pytest.mark.vcr 8 | @pytest.mark.parametrize( 9 | "nd_llm_cls", 10 | [ 11 | (_ndllm_factory(_NDClientTarget.ROUTER)), 12 | (_ndllm_factory(_NDClientTarget.INVOKER)), 13 | ], 14 | ) 15 | def test_model_select_callback(llm_base_callback_handler, nd_llm_cls): 16 | """ 17 | This tests that the on_model_select method is called when a model is selected. 18 | """ 19 | 20 | nd_llm = nd_llm_cls( 21 | llm_configs=[ 22 | NDLLMProviders.GPT_4_TURBO_PREVIEW, 23 | NDLLMProviders.GPT_3_5_TURBO, 24 | ], 25 | callbacks=[llm_base_callback_handler], 26 | ) 27 | nd_llm.model_select( 28 | messages=[{"role": "user", "content": "Hello, what's your name?"}] 29 | ) 30 | assert llm_base_callback_handler.on_model_select_called 31 | 32 | 33 | @pytest.mark.vcr 34 | def test_latency_tracking_callback(llm_base_callback_handler, nd_invoker_cls): 35 | """ 36 | This tests that latency tracking is enabled and the on_latency_tracking method is called. 37 | """ 38 | nd_llm = nd_invoker_cls( 39 | llm_configs=[ 40 | NDLLMProviders.GPT_4_TURBO_PREVIEW, 41 | NDLLMProviders.GPT_3_5_TURBO, 42 | ], 43 | callbacks=[llm_base_callback_handler], 44 | latency_tracking=True, 45 | ) 46 | nd_llm.invoke( 47 | messages=[{"role": "user", "content": "Hello, what's your name?"}] 48 | ) 49 | assert llm_base_callback_handler.on_latency_tracking_called 50 | 51 | 52 | @pytest.mark.vcr 53 | def test_llm_start_callback(llm_base_callback_handler, nd_invoker_cls): 54 | """ 55 | This tests that callback handler is passed along correctly to 56 | the langchain LLM class, by checking if the on_llm_start method is called. 57 | """ 58 | nd_llm = nd_invoker_cls( 59 | llm_configs=[ 60 | NDLLMProviders.GPT_4_TURBO_PREVIEW, 61 | NDLLMProviders.GPT_3_5_TURBO, 62 | ], 63 | callbacks=[llm_base_callback_handler], 64 | ) 65 | nd_llm.invoke(messages=[{"role": "user", "content": "Tell me a joke."}]) 66 | assert llm_base_callback_handler.on_llm_start_called 67 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/test_embedding_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from notdiamond.exceptions import UnsupportedEmbeddingProvider 4 | from notdiamond.llms.config import EmbeddingConfig 5 | 6 | 7 | def test_supported_embedding_provider(): 8 | config = EmbeddingConfig(provider="openai", model="text-embedding-3-small") 9 | assert config.provider == "openai" 10 | assert config.model == "text-embedding-3-small" 11 | 12 | config = EmbeddingConfig(provider="mistral", model="mistral-embed") 13 | assert config.provider == "mistral" 14 | assert config.model == "mistral-embed" 15 | 16 | config = EmbeddingConfig(provider="cohere", model="embed-english-v3.0") 17 | assert config.provider == "cohere" 18 | assert config.model == "embed-english-v3.0" 19 | 20 | 21 | def test_unsupported_embedding_model(): 22 | with pytest.raises(UnsupportedEmbeddingProvider): 23 | EmbeddingConfig(provider="openai", model="gpt-71") 24 | 25 | 26 | def test_config_from_string(): 27 | config = EmbeddingConfig.from_string("openai/text-embedding-3-small") 28 | assert config.provider == "openai" 29 | assert config.model == "text-embedding-3-small" 30 | -------------------------------------------------------------------------------- /tests/test_components/test_llms/test_provider.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | import pytest 4 | 5 | from notdiamond import LLMConfig 6 | from notdiamond.exceptions import UnsupportedLLMProvider 7 | 8 | sys.path.append("../") 9 | 10 | 11 | def test_supported_llm_provider(): 12 | openai = LLMConfig(provider="openai", model="gpt-3.5-turbo") 13 | assert openai.provider == "openai" 14 | assert openai.model == "gpt-3.5-turbo" 15 | 16 | openai = LLMConfig(provider="anthropic", model="claude-2.1") 17 | assert openai.provider == "anthropic" 18 | assert openai.model == "claude-2.1" 19 | 20 | openai = LLMConfig(provider="google", model="gemini-pro") 21 | assert openai.provider == "google" 22 | assert openai.model == "gemini-pro" 23 | 24 | 25 | def test_unsupported_model(): 26 | with pytest.raises(UnsupportedLLMProvider): 27 | LLMConfig(provider="openai", model="gpt-71") 28 | 29 | 30 | def test_prepare_for_request(): 31 | llm_provider = LLMConfig( 32 | provider="openai", 33 | model="gpt-3.5-turbo", 34 | context_length=1, 35 | input_price=0.1, 36 | output_price=0.4, 37 | latency=100, 38 | ) 39 | request = llm_provider.prepare_for_request() 40 | assert request == { 41 | "provider": llm_provider.provider, 42 | "model": llm_provider.model, 43 | "is_custom": llm_provider.is_custom, 44 | "context_length": llm_provider.context_length, 45 | "input_price": llm_provider.input_price, 46 | "output_price": llm_provider.output_price, 47 | "latency": llm_provider.latency, 48 | } 49 | 50 | 51 | def test_existing_openrouter_model(): 52 | llm_provider = LLMConfig(provider="mistral", model="mistral-large-latest") 53 | assert llm_provider.openrouter_model == "mistralai/mistral-large" 54 | 55 | llm_provider = LLMConfig( 56 | provider="anthropic", model="claude-3-opus-20240229" 57 | ) 58 | assert llm_provider.openrouter_model == "anthropic/claude-3-opus" 59 | 60 | 61 | def test_not_existing_openrouter_model(): 62 | llm_provider = LLMConfig(provider="openai", model="gpt-4-0125-preview") 63 | result = llm_provider.openrouter_model 64 | assert result is None 65 | -------------------------------------------------------------------------------- /tests/test_components/test_utils.py: -------------------------------------------------------------------------------- 1 | """Tests for utility functions in _utils.py""" 2 | 3 | import pytest 4 | 5 | from notdiamond._utils import token_counter 6 | 7 | 8 | class TestTokenCounter: 9 | """Test class for token_counter function""" 10 | 11 | def test_token_counter_with_text(self): 12 | """Test token_counter with text input""" 13 | text = "This is a simple test sentence." 14 | count = token_counter(model="gpt-4", text=text) 15 | assert isinstance(count, int) 16 | assert count > 0 17 | assert count <= len(text) # Tokens should be fewer than characters 18 | 19 | def test_token_counter_with_messages(self): 20 | """Test token_counter with messages input""" 21 | messages = [ 22 | {"role": "user", "content": "Hello, how are you?"}, 23 | {"role": "assistant", "content": "I'm doing well, thank you for asking!"} 24 | ] 25 | count = token_counter(model="gpt-3.5-turbo", messages=messages) 26 | assert isinstance(count, int) 27 | assert count > 0 28 | 29 | # Verify that computing tokens from text and from messages gives consistent results 30 | combined_text = "Hello, how are you?I'm doing well, thank you for asking!" 31 | text_count = token_counter(model="gpt-3.5-turbo", text=combined_text) 32 | assert count >= text_count # Messages might add a few tokens for role markers 33 | 34 | def test_token_counter_different_models(self): 35 | """Test token_counter with different models""" 36 | text = "This is a test of token counting across different models." 37 | 38 | gpt4_count = token_counter(model="gpt-4", text=text) 39 | gpt35_count = token_counter(model="gpt-3.5-turbo", text=text) 40 | 41 | # Token counts should be similar for same text across different models 42 | assert abs(gpt4_count - gpt35_count) <= 2 43 | 44 | def test_token_counter_fallback(self): 45 | """Test token_counter fallback mechanism with unknown model""" 46 | text = "Testing the fallback mechanism with an unknown model." 47 | 48 | # Test with a model name that won't have a specific tokenizer 49 | count = token_counter(model="unknown-model", text=text) 50 | assert isinstance(count, int) 51 | assert count > 0 52 | 53 | # Compare with a known model's count 54 | known_model_count = token_counter(model="gpt-4", text=text) 55 | # The counts should be reasonably similar 56 | assert abs(count - known_model_count) <= 5 57 | 58 | def test_token_counter_error_handling(self): 59 | """Test token_counter error handling""" 60 | # Test with neither text nor messages 61 | with pytest.raises(ValueError): 62 | token_counter(model="gpt-4") -------------------------------------------------------------------------------- /tests/test_documentation/cassettes/test_function_calling/test_function_calling_via_rest_api.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "system", "content": "You are a mathematician with 4 | tools at your disposal."}, {"role": "assistant", "content": "How can I help 5 | you today?"}, {"role": "user", "content": "What is 234 + 82234?"}], "llm_providers": 6 | [{"provider": "openai", "model": "gpt-4-1106-preview"}, {"provider": "anthropic", 7 | "model": "claude-3-opus-20240229"}], "tools": [{"type": "function", "function": 8 | {"name": "add", "description": "Adds a and b.", "parameters": {"type": "object", 9 | "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}, "required": 10 | ["a", "b"]}}}, {"type": "function", "function": {"name": "multiply", "description": 11 | "Multiplies a and b.", "parameters": {"type": "object", "properties": {"a": 12 | {"type": "integer"}, "b": {"type": "integer"}}, "required": ["a", "b"]}}}]}' 13 | headers: 14 | Accept-Encoding: 15 | - gzip, deflate, zstd 16 | Connection: 17 | - keep-alive 18 | Content-Length: 19 | - '793' 20 | User-Agent: 21 | - python-requests/2.32.3 22 | accept: 23 | - application/json 24 | content-type: 25 | - application/json 26 | method: POST 27 | uri: https://staging-api.notdiamond.ai/v2/optimizer/hashModelSelect 28 | response: 29 | body: 30 | string: '{"providers":[{"provider":"openai","model":"gpt-4-1106-preview"}],"session_id":"1c01fac2-0bba-4d25-b9ec-ecc035e73dc4"}' 31 | headers: 32 | CF-RAY: 33 | - 925d1ba3ce02d0ec-SOF 34 | Connection: 35 | - keep-alive 36 | Content-Type: 37 | - application/json 38 | Date: 39 | - Tue, 25 Mar 2025 08:41:14 GMT 40 | Server: 41 | - cloudflare 42 | Transfer-Encoding: 43 | - chunked 44 | alt-svc: 45 | - h3=":443"; ma=86400 46 | cf-cache-status: 47 | - DYNAMIC 48 | content-length: 49 | - '118' 50 | rndr-id: 51 | - f828ef91-15d2-4403 52 | vary: 53 | - Accept-Encoding 54 | x-render-origin-server: 55 | - uvicorn 56 | status: 57 | code: 200 58 | message: OK 59 | version: 1 60 | -------------------------------------------------------------------------------- /tests/test_documentation/cassettes/test_getting_started/test_model_select.yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "system", "content": "You are a world class software 4 | developer."}, {"role": "user", "content": "Write a merge sort in Python"}], 5 | "llm_providers": [{"provider": "openai", "model": "gpt-3.5-turbo", "is_custom": 6 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 7 | null}, {"provider": "openai", "model": "gpt-4", "is_custom": false, "context_length": 8 | null, "input_price": null, "output_price": null, "latency": null}, {"provider": 9 | "openai", "model": "gpt-4-1106-preview", "is_custom": false, "context_length": 10 | null, "input_price": null, "output_price": null, "latency": null}, {"provider": 11 | "openai", "model": "gpt-4-turbo-preview", "is_custom": false, "context_length": 12 | null, "input_price": null, "output_price": null, "latency": null}, {"provider": 13 | "anthropic", "model": "claude-3-haiku-20240307", "is_custom": false, "context_length": 14 | null, "input_price": null, "output_price": null, "latency": null}, {"provider": 15 | "anthropic", "model": "claude-3-sonnet-20240229", "is_custom": false, "context_length": 16 | null, "input_price": null, "output_price": null, "latency": null}, {"provider": 17 | "anthropic", "model": "claude-3-opus-20240229", "is_custom": false, "context_length": 18 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 19 | "accuracy", "max_model_depth": 7, "hash_content": false, "tradeoff": "cost"}' 20 | headers: 21 | Accept: 22 | - '*/*' 23 | Accept-Encoding: 24 | - gzip, deflate, zstd 25 | Connection: 26 | - keep-alive 27 | Content-Length: 28 | - '1377' 29 | User-Agent: 30 | - Python-SDK/0.3.41 31 | content-type: 32 | - application/json 33 | method: POST 34 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 35 | response: 36 | body: 37 | string: '{"providers":[{"provider":"openai","model":"gpt-4-1106-preview"}],"session_id":"68a6cf92-0acc-4608-8ddd-4b8dadae3aec"}' 38 | headers: 39 | CF-RAY: 40 | - 925cfdab7adec30a-VIE 41 | Connection: 42 | - keep-alive 43 | Content-Type: 44 | - application/json 45 | Date: 46 | - Tue, 25 Mar 2025 08:20:46 GMT 47 | Server: 48 | - cloudflare 49 | Transfer-Encoding: 50 | - chunked 51 | alt-svc: 52 | - h3=":443"; ma=86400 53 | cf-cache-status: 54 | - DYNAMIC 55 | content-length: 56 | - '118' 57 | rndr-id: 58 | - 709cef38-95a6-4614 59 | vary: 60 | - Accept-Encoding 61 | x-render-origin-server: 62 | - uvicorn 63 | status: 64 | code: 200 65 | message: OK 66 | version: 1 67 | -------------------------------------------------------------------------------- /tests/test_documentation/test_fallback_and_custom.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from openai import OpenAI 3 | 4 | from notdiamond import settings 5 | from notdiamond.llms.client import NotDiamond 6 | 7 | 8 | @pytest.mark.vcr 9 | def test_fallback_model(): 10 | llm_configs = [ 11 | "openai/gpt-3.5-turbo", 12 | "openai/gpt-4", 13 | "openai/gpt-4-1106-preview", 14 | "openai/gpt-4-turbo-preview", 15 | "anthropic/claude-2.1", 16 | "anthropic/claude-3-sonnet-20240229", 17 | "anthropic/claude-3-opus-20240229", 18 | "google/gemini-pro", 19 | ] 20 | 21 | # Setting a fallback model 22 | NotDiamond( 23 | llm_configs=llm_configs, 24 | default="openai/gpt-4-turbo-preview", # The model from llm_configs you want to fallback to 25 | ) 26 | 27 | 28 | @pytest.mark.vcr 29 | def test_set_max_model_depth(): 30 | llm_configs = [ 31 | "openai/gpt-3.5-turbo", 32 | "openai/gpt-4", 33 | "openai/gpt-4-1106-preview", 34 | "openai/gpt-4-turbo-preview", 35 | "anthropic/claude-2.1", 36 | "anthropic/claude-3-sonnet-20240229", 37 | "anthropic/claude-3-opus-20240229", 38 | "google/gemini-pro", 39 | ] 40 | 41 | # Setting recommendation depth 42 | NotDiamond( 43 | llm_configs=llm_configs, 44 | max_model_depth=3, # The maximum depth in the recommendation ranking to consider 45 | ) 46 | 47 | 48 | @pytest.mark.vcr 49 | def test_custom_logic(): 50 | # Define the string that will be routed to the best LLM 51 | prompt = "You are a world class software developer. Write a merge sort in Python." 52 | 53 | # Define the available LLMs you'd like to route between 54 | llm_configs = [ 55 | "openai/gpt-3.5-turbo", 56 | ] 57 | 58 | # Create the NDLLM object -> like a 'meta-LLM' combining all of the specified models 59 | nd_llm = NotDiamond( 60 | llm_configs=llm_configs, 61 | tradeoff="cost", 62 | ) # Define preferences 63 | 64 | session_id, provider = nd_llm.model_select(messages=prompt) 65 | 66 | print(session_id) 67 | print(provider) 68 | 69 | client = OpenAI(api_key=settings.OPENAI_API_KEY) 70 | 71 | max_retries = 3 72 | 73 | if provider.model == "gpt-3.5-turbo": 74 | for _ in range(max_retries): 75 | try: 76 | chat_completion = client.chat.completions.create( 77 | messages=[ 78 | { 79 | "role": "user", 80 | "content": prompt, 81 | } 82 | ], 83 | model="gpt-3.5-turbo", 84 | ) 85 | return chat_completion.choices[0].message.content 86 | except Exception as e: 87 | print(e) 88 | continue 89 | -------------------------------------------------------------------------------- /tests/test_documentation/test_langchain.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from notdiamond.llms.client import NotDiamond 4 | 5 | 6 | @pytest.mark.vcr 7 | def test_streaming(): 8 | messages = [ 9 | { 10 | "role": "system", 11 | "content": "You are a world class software developer.", 12 | }, 13 | {"role": "user", "content": "Write merge sort in Python."}, 14 | ] 15 | 16 | chat = NotDiamond( 17 | llm_configs=[ 18 | "openai/gpt-3.5-turbo", 19 | "openai/gpt-4", 20 | "anthropic/claude-2.1", 21 | "google/gemini-pro", 22 | ] 23 | ) 24 | 25 | for chunk in chat.stream(messages): 26 | print(chunk.content, end="", flush=True) 27 | -------------------------------------------------------------------------------- /tests/test_documentation/test_openrouter.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import pytest 5 | import requests 6 | from dotenv import load_dotenv 7 | 8 | from notdiamond.llms.client import NotDiamond 9 | 10 | load_dotenv(os.getcwd() + "/.env") 11 | 12 | 13 | @pytest.mark.vcr 14 | def test_openrouter_integration(): 15 | openrouter_api_key = os.getenv("OPENROUTER_API_KEY", default="") 16 | messages = [ 17 | { 18 | "role": "system", 19 | "content": "You are a world class software developer.", 20 | }, 21 | {"role": "user", "content": "Write a merge sort in Python"}, 22 | ] 23 | 24 | llm_configs = [ 25 | "openai/gpt-3.5-turbo", 26 | "openai/gpt-4", 27 | "openai/gpt-4-1106-preview", 28 | "openai/gpt-4-turbo-preview", 29 | "anthropic/claude-3-haiku-20240307", 30 | "anthropic/claude-3-sonnet-20240229", 31 | "anthropic/claude-3-opus-20240229", 32 | ] 33 | 34 | nd_llm = NotDiamond(llm_configs=llm_configs) 35 | session_id, provider = nd_llm.model_select(messages=messages) 36 | 37 | response = requests.post( 38 | url="https://openrouter.ai/api/v1/chat/completions", 39 | headers={ 40 | "Authorization": f"Bearer {openrouter_api_key}", 41 | }, 42 | data=json.dumps( 43 | {"model": provider.openrouter_model, "messages": messages} 44 | ), 45 | ) 46 | 47 | print(response) 48 | -------------------------------------------------------------------------------- /tests/test_documentation/test_personalization.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from notdiamond.llms.client import NotDiamond 4 | from notdiamond.metrics.metric import Metric 5 | 6 | 7 | @pytest.mark.vcr 8 | def test_personalization(): 9 | prompt = [ 10 | { 11 | "role": "user", 12 | "content": "You are a world class software developer. Write a merge sort in Python.", 13 | } 14 | ] 15 | 16 | llm_configs = [ 17 | "openai/gpt-3.5-turbo", 18 | "openai/gpt-4", 19 | "openai/gpt-4-1106-preview", 20 | "openai/gpt-4-turbo-preview", 21 | "anthropic/claude-2.1", 22 | "anthropic/claude-3-sonnet-20240229", 23 | "anthropic/claude-3-opus-20240229", 24 | "google/gemini-pro", 25 | ] 26 | 27 | nd_llm = NotDiamond( 28 | llm_configs=llm_configs, 29 | preference_id="5c5af286-b715-4d8b-8cf9-151230ef96a3", 30 | ) 31 | metric = Metric("accuracy") 32 | 33 | result, session_id, provider = nd_llm.invoke(prompt, metric=metric) 34 | 35 | # Application logic... 36 | # Let's assume the result from the LLM achieved the outcome you're looking for! 37 | 38 | score = metric.feedback( 39 | session_id=session_id, llm_config=provider, value=1 40 | ) 41 | print(score) 42 | -------------------------------------------------------------------------------- /tests/test_documentation/test_structured_output.py: -------------------------------------------------------------------------------- 1 | from json import JSONDecoder 2 | 3 | import pytest 4 | from langchain_core.exceptions import OutputParserException 5 | from pydantic import BaseModel, Field 6 | 7 | from notdiamond.llms.client import NotDiamond 8 | 9 | 10 | @pytest.mark.vcr 11 | def test_structured_output(): 12 | class LanguageChoice(BaseModel): 13 | language: str = Field( 14 | description="The programming language of choice." 15 | ) 16 | reason: str = Field( 17 | description="The reason to pick the programming language." 18 | ) 19 | 20 | messages = [ 21 | { 22 | "role": "system", 23 | "content": "You are a world class software developer.", 24 | }, 25 | { 26 | "role": "user", 27 | "content": "What language would you suggest for developing a web application?", 28 | }, 29 | ] 30 | 31 | llm_configs = [ 32 | "openai/gpt-3.5-turbo", 33 | "openai/gpt-4", 34 | "openai/gpt-4-1106-preview", 35 | "openai/gpt-4-turbo-preview", 36 | "anthropic/claude-3-opus-20240229", 37 | ] 38 | 39 | nd_llm = NotDiamond(llm_configs=llm_configs) 40 | 41 | def extract_json_objects(text, decoder=JSONDecoder()): 42 | """Find JSON objects in text, and yield the decoded JSON data 43 | 44 | Does not attempt to look for JSON arrays, text, or other JSON types outside 45 | of a parent JSON object. 46 | 47 | """ 48 | pos = 0 49 | while True: 50 | match = text.find("{", pos) 51 | if match == -1: 52 | break 53 | try: 54 | result, index = decoder.raw_decode(text[match:]) 55 | yield result 56 | pos = match + index 57 | except ValueError: 58 | pos = match + 1 59 | 60 | try: 61 | result, session_id, _ = nd_llm.invoke( 62 | messages=messages, response_model=LanguageChoice 63 | ) 64 | except OutputParserException: 65 | result, session_id, _ = nd_llm.invoke(messages=messages) 66 | json_obj = list(extract_json_objects(result.content)) 67 | assert len(json_obj) == 1 68 | json_response = json_obj[0] 69 | assert len(json_response) == 2 70 | assert "language" in json_response 71 | assert "reason" in json_response 72 | 73 | print(result) 74 | 75 | 76 | @pytest.mark.vcr 77 | def test_structured_output_streaming(): 78 | class LanguageChoice(BaseModel): 79 | language: str = Field( 80 | description="The programming language of choice." 81 | ) 82 | reason: str = Field( 83 | description="The reason to pick the programming language." 84 | ) 85 | 86 | messages = [ 87 | { 88 | "role": "system", 89 | "content": "You are a world class software developer.", 90 | }, 91 | { 92 | "role": "user", 93 | "content": "What language would you suggest for developing a web application?", 94 | }, 95 | ] 96 | 97 | llm_configs = [ 98 | "openai/gpt-3.5-turbo", 99 | "openai/gpt-4", 100 | "openai/gpt-4-1106-preview", 101 | "openai/gpt-4-turbo-preview", 102 | "anthropic/claude-3-opus-20240229", 103 | ] 104 | 105 | nd_llm = NotDiamond(llm_configs=llm_configs) 106 | 107 | for chunk in nd_llm.stream( 108 | messages=messages, response_model=LanguageChoice 109 | ): 110 | print(chunk) 111 | -------------------------------------------------------------------------------- /tests/test_llm_calls/test_cohere.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from notdiamond.llms.client import NotDiamond 4 | from notdiamond.llms.providers import NDLLMProviders 5 | 6 | 7 | @pytest.mark.vcr 8 | @pytest.mark.longrun 9 | class Test_Cohere: 10 | def test_cohere_command_r_plus_with_tool_calling(self, tools_fixture): 11 | provider = NDLLMProviders.COMMAND_R_PLUS 12 | provider.kwargs = {"max_tokens": 200} 13 | nd_llm = NotDiamond( 14 | llm_configs=[provider], latency_tracking=False, hash_content=True 15 | ) 16 | nd_llm = nd_llm.bind_tools(tools_fixture) 17 | result, session_id, _ = nd_llm.invoke( 18 | [{"role": "user", "content": "How much is 3 + 5?"}] 19 | ) 20 | 21 | assert len(result.tool_calls) == 1 22 | assert result.tool_calls[0]["name"] == "add_fct" 23 | 24 | def test_cohere_command_r_plus_response_model(self, response_model): 25 | provider = NDLLMProviders.COMMAND_R_PLUS 26 | provider.kwargs = {"max_tokens": 200} 27 | nd_llm = NotDiamond( 28 | llm_configs=[provider], latency_tracking=False, hash_content=True 29 | ) 30 | result, _, _ = nd_llm.invoke( 31 | [{"role": "user", "content": "Tell me a joke"}], 32 | response_model=response_model, 33 | ) 34 | 35 | assert isinstance(result, response_model) 36 | assert result.setup 37 | assert result.punchline 38 | 39 | def test_cohere_command_r_with_tool_calling(self, tools_fixture): 40 | provider = NDLLMProviders.COMMAND_R 41 | provider.kwargs = {"max_tokens": 200} 42 | nd_llm = NotDiamond( 43 | llm_configs=[provider], latency_tracking=False, hash_content=True 44 | ) 45 | nd_llm = nd_llm.bind_tools(tools_fixture) 46 | result, session_id, _ = nd_llm.invoke( 47 | [{"role": "user", "content": "How much is 3 + 5?"}] 48 | ) 49 | 50 | assert len(result.tool_calls) == 1 51 | assert result.tool_calls[0]["name"] == "add_fct" 52 | 53 | def test_cohere_command_r_response_model(self, response_model): 54 | provider = NDLLMProviders.COMMAND_R 55 | provider.kwargs = {"max_tokens": 200} 56 | nd_llm = NotDiamond( 57 | llm_configs=[provider], latency_tracking=False, hash_content=True 58 | ) 59 | result, _, _ = nd_llm.invoke( 60 | [{"role": "user", "content": "Tell me a joke"}], 61 | response_model=response_model, 62 | ) 63 | 64 | assert isinstance(result, response_model) 65 | assert result.setup 66 | assert result.punchline 67 | -------------------------------------------------------------------------------- /tests/test_llm_calls/test_openai_o1.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from notdiamond.llms.client import NotDiamond 4 | from notdiamond.llms.providers import NDLLMProviders 5 | 6 | test_providers = [ 7 | provider 8 | for provider in NDLLMProviders 9 | if provider.provider == "openai" and provider.model[:2] == "o1" 10 | ] 11 | 12 | 13 | @pytest.mark.vcr 14 | @pytest.mark.longrun 15 | @pytest.mark.parametrize("provider", test_providers) 16 | def test_o1_with_system_prompt(provider): 17 | nd_llm = NotDiamond( 18 | llm_configs=[provider], latency_tracking=False, hash_content=True 19 | ) 20 | result, session_id, _ = nd_llm.invoke( 21 | [ 22 | {"role": "system", "content": "You are a funny AI"}, 23 | {"role": "user", "content": "Tell me a joke"}, 24 | ], 25 | ) 26 | 27 | assert session_id != "NO-SESSION-ID" 28 | assert len(result.content) > 0 29 | -------------------------------------------------------------------------------- /tests/test_llm_calls/test_perplexity.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from notdiamond.llms.client import NotDiamond 4 | from notdiamond.llms.providers import NDLLMProviders 5 | from notdiamond.metrics.metric import Metric 6 | 7 | 8 | @pytest.mark.vcr 9 | @pytest.mark.longrun 10 | class Test_Perplexity_LLMs: 11 | def test_sonar(self): 12 | provider = NDLLMProviders.SONAR 13 | provider.kwargs = {"max_tokens": 10} 14 | nd_llm = NotDiamond( 15 | llm_configs=[provider], 16 | latency_tracking=False, 17 | hash_content=False, 18 | ) 19 | result, session_id, _ = nd_llm.invoke( 20 | messages=[{"role": "user", "content": "How much is 3 + 5?"}], 21 | metric=Metric("accuracy"), 22 | ) 23 | 24 | assert session_id != "NO-SESSION-ID" 25 | assert len(result.content) > 0 26 | -------------------------------------------------------------------------------- /tests/test_llm_calls/test_replicate.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from notdiamond.llms.client import NotDiamond 4 | from notdiamond.llms.providers import NDLLMProviders 5 | from notdiamond.metrics.metric import Metric 6 | 7 | 8 | @pytest.mark.vcr 9 | @pytest.mark.longrun 10 | @pytest.mark.skip("Currently failing, need to debug") 11 | class Test_Replicate_LLMs: 12 | def test_mistral_7b_instruct_v02(self): 13 | provider = NDLLMProviders.REPLICATE_MISTRAL_7B_INSTRUCT_V0_2 14 | provider.kwargs = {"max_tokens": 10} 15 | nd_llm = NotDiamond( 16 | llm_configs=[provider], latency_tracking=False, hash_content=True 17 | ) 18 | result, session_id, _ = nd_llm.invoke( 19 | messages=[{"role": "user", "content": "How much is 3 + 5?"}], 20 | metric=Metric("accuracy"), 21 | ) 22 | 23 | assert session_id != "NO-SESSION-ID" 24 | assert len(result.content) > 0 25 | 26 | def test_mixtral_8x7b_instruct_v01(self): 27 | provider = NDLLMProviders.REPLICATE_MIXTRAL_8X7B_INSTRUCT_V0_1 28 | provider.kwargs = {"max_tokens": 10} 29 | nd_llm = NotDiamond( 30 | llm_configs=[provider], latency_tracking=False, hash_content=True 31 | ) 32 | result, session_id, _ = nd_llm.invoke( 33 | messages=[{"role": "user", "content": "How much is 3 + 5?"}], 34 | metric=Metric("accuracy"), 35 | ) 36 | 37 | assert session_id != "NO-SESSION-ID" 38 | assert len(result.content) > 0 39 | 40 | def test_llama_3_70b_instruct(self): 41 | provider = NDLLMProviders.REPLICATE_META_LLAMA_3_70B_INSTRUCT 42 | provider.kwargs = {"max_tokens": 10} 43 | nd_llm = NotDiamond( 44 | llm_configs=[provider], 45 | latency_tracking=False, 46 | hash_content=False, 47 | ) 48 | result, session_id, _ = nd_llm.invoke( 49 | messages=[{"role": "user", "content": "How much is 3 + 5?"}], 50 | metric=Metric("accuracy"), 51 | ) 52 | 53 | assert session_id != "NO-SESSION-ID" 54 | assert len(result.content) > 0 55 | 56 | def test_llama_3_8b_instruct_with_prompt_template(self): 57 | provider = NDLLMProviders.REPLICATE_META_LLAMA_3_8B_INSTRUCT 58 | provider.kwargs = {"max_tokens": 10} 59 | nd_llm = NotDiamond( 60 | llm_configs=[provider], 61 | latency_tracking=False, 62 | hash_content=False, 63 | ) 64 | result, session_id, _ = nd_llm.invoke( 65 | messages=[{"role": "user", "content": "How much is 3 + 5?"}], 66 | metric=Metric("accuracy"), 67 | ) 68 | 69 | assert session_id != "NO-SESSION-ID" 70 | assert len(result.content) > 0 71 | 72 | def test_llama_3_1_405b_instruct(self): 73 | provider = NDLLMProviders.REPLICATE_META_LLAMA_3_1_405B_INSTRUCT 74 | provider.kwargs = {"max_tokens": 10} 75 | nd_llm = NotDiamond( 76 | llm_configs=[provider], 77 | latency_tracking=False, 78 | hash_content=False, 79 | ) 80 | result, session_id, _ = nd_llm.invoke( 81 | messages=[{"role": "user", "content": "How much is 3 + 5?"}], 82 | metric=Metric("accuracy"), 83 | ) 84 | 85 | assert session_id != "NO-SESSION-ID" 86 | assert len(result.content) > 0 87 | -------------------------------------------------------------------------------- /tests/test_toolkit/cassettes/test_retry/test_retry_wrapper[anthropic].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "Hello, how 4 | are you?"}], "model": "claude-3-5-haiku-20241022"}' 5 | headers: 6 | accept: 7 | - application/json 8 | accept-encoding: 9 | - gzip, deflate 10 | anthropic-version: 11 | - '2023-06-01' 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - '124' 16 | content-type: 17 | - application/json 18 | host: 19 | - api.anthropic.com 20 | user-agent: 21 | - Anthropic/Python 0.39.0 22 | x-api-key: 23 | - MASKED-API-KEY-FOR-TESTING 24 | x-stainless-arch: 25 | - arm64 26 | x-stainless-async: 27 | - 'false' 28 | x-stainless-lang: 29 | - python 30 | x-stainless-os: 31 | - MacOS 32 | x-stainless-package-version: 33 | - 0.39.0 34 | x-stainless-retry-count: 35 | - '0' 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.11.10 40 | method: POST 41 | uri: https://api.anthropic.com/v1/messages 42 | response: 43 | body: 44 | string: '{"id":"msg_017E8Qpatk7sjidTqfJiRiru","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''m 45 | doing well, thank you for asking! How are you today? Is there anything I can 46 | help you with?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":13,"output_tokens":27}}' 47 | headers: 48 | CF-Cache-Status: 49 | - DYNAMIC 50 | CF-RAY: 51 | - 8ed7549aaa9567de-MIA 52 | Connection: 53 | - keep-alive 54 | Content-Type: 55 | - application/json 56 | Date: 57 | - Thu, 05 Dec 2024 22:04:12 GMT 58 | Server: 59 | - cloudflare 60 | Transfer-Encoding: 61 | - chunked 62 | X-Robots-Tag: 63 | - none 64 | anthropic-ratelimit-input-tokens-limit: 65 | - '400000' 66 | anthropic-ratelimit-input-tokens-remaining: 67 | - '400000' 68 | anthropic-ratelimit-input-tokens-reset: 69 | - '2024-12-05T22:04:12Z' 70 | anthropic-ratelimit-output-tokens-limit: 71 | - '80000' 72 | anthropic-ratelimit-output-tokens-remaining: 73 | - '80000' 74 | anthropic-ratelimit-output-tokens-reset: 75 | - '2024-12-05T22:04:12Z' 76 | anthropic-ratelimit-requests-limit: 77 | - '4000' 78 | anthropic-ratelimit-requests-remaining: 79 | - '3999' 80 | anthropic-ratelimit-requests-reset: 81 | - '2024-12-05T22:04:11Z' 82 | anthropic-ratelimit-tokens-limit: 83 | - '480000' 84 | anthropic-ratelimit-tokens-remaining: 85 | - '480000' 86 | anthropic-ratelimit-tokens-reset: 87 | - '2024-12-05T22:04:12Z' 88 | content-length: 89 | - '335' 90 | request-id: 91 | - req_01Kvba1zUfqLadADM3McoKuf 92 | via: 93 | - 1.1 google 94 | status: 95 | code: 200 96 | message: OK 97 | version: 1 98 | -------------------------------------------------------------------------------- /tests/test_toolkit/cassettes/test_retry/test_retry_wrapper[openai].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model": 4 | "gpt-4o-mini"}' 5 | headers: 6 | accept: 7 | - application/json 8 | accept-encoding: 9 | - gzip, deflate 10 | connection: 11 | - keep-alive 12 | content-length: 13 | - '90' 14 | content-type: 15 | - application/json 16 | host: 17 | - api.openai.com 18 | user-agent: 19 | - OpenAI/Python 1.55.0 20 | x-stainless-arch: 21 | - arm64 22 | x-stainless-async: 23 | - 'false' 24 | x-stainless-lang: 25 | - python 26 | x-stainless-os: 27 | - MacOS 28 | x-stainless-package-version: 29 | - 1.55.0 30 | x-stainless-retry-count: 31 | - '0' 32 | x-stainless-runtime: 33 | - CPython 34 | x-stainless-runtime-version: 35 | - 3.11.10 36 | method: POST 37 | uri: https://api.openai.com/v1/chat/completions 38 | response: 39 | body: 40 | string: "{\n \"id\": \"chatcmpl-AbEF8ClecDZCPMUjm7boOu2lFSDVl\",\n \"object\": 41 | \"chat.completion\",\n \"created\": 1733436250,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n 42 | \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": 43 | \"assistant\",\n \"content\": \"Hello! I'm just a computer program, 44 | so I don't have feelings, but I'm here and ready to help you. How can I assist 45 | you today?\",\n \"refusal\": null\n },\n \"logprobs\": null,\n 46 | \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 47 | 13,\n \"completion_tokens\": 30,\n \"total_tokens\": 43,\n \"prompt_tokens_details\": 48 | {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": 49 | {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 50 | 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"system_fingerprint\": 51 | \"fp_0705bf87c0\"\n}\n" 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 8ed75494abf8db2d-MIA 57 | Connection: 58 | - keep-alive 59 | Content-Type: 60 | - application/json 61 | Date: 62 | - Thu, 05 Dec 2024 22:04:11 GMT 63 | Server: 64 | - cloudflare 65 | Set-Cookie: 66 | - __cf_bm=GA2qg.kK2zm.1O5mc33iZL0OJ9qc4lZs.2E.kwVptfQ-1733436251-1.0.1.1-yG7_72uvTjgx5GWzsLPfFlmwGYhO5HNxBqoU98ExQIswRbZ9mMJq2amr5mcayHgeVKhyfRulb8iUl_umvH54Rg; 67 | path=/; expires=Thu, 05-Dec-24 22:34:11 GMT; domain=.api.openai.com; HttpOnly; 68 | Secure; SameSite=None 69 | - _cfuvid=TwJ5E0YXwwWinJhkalH3gS5WYUKa6w30Fkfh_3x_OaU-1733436251203-0.0.1.1-604800000; 70 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 71 | Transfer-Encoding: 72 | - chunked 73 | X-Content-Type-Options: 74 | - nosniff 75 | access-control-expose-headers: 76 | - X-Request-ID 77 | alt-svc: 78 | - h3=":443"; ma=86400 79 | content-length: 80 | - '876' 81 | openai-organization: 82 | - not-diamond-bdz8cg 83 | openai-processing-ms: 84 | - '776' 85 | openai-version: 86 | - '2020-10-01' 87 | strict-transport-security: 88 | - max-age=31536000; includeSubDomains; preload 89 | x-ratelimit-limit-requests: 90 | - '30000' 91 | x-ratelimit-limit-tokens: 92 | - '150000000' 93 | x-ratelimit-remaining-requests: 94 | - '29999' 95 | x-ratelimit-remaining-tokens: 96 | - '149999978' 97 | x-ratelimit-reset-requests: 98 | - 2ms 99 | x-ratelimit-reset-tokens: 100 | - 0s 101 | x-request-id: 102 | - req_74329deb7921fcd31e0c9ce9b89a0ede 103 | status: 104 | code: 200 105 | message: OK 106 | version: 1 107 | -------------------------------------------------------------------------------- /tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_async[async-anthropic].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "Hello, how 4 | are you?"}], "model": "claude-3-5-haiku-20241022"}' 5 | headers: 6 | accept: 7 | - application/json 8 | accept-encoding: 9 | - gzip, deflate 10 | anthropic-version: 11 | - '2023-06-01' 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - '124' 16 | content-type: 17 | - application/json 18 | host: 19 | - api.anthropic.com 20 | user-agent: 21 | - AsyncAnthropic/Python 0.39.0 22 | x-api-key: 23 | - MASKED-API-KEY-FOR-TESTING 24 | x-stainless-arch: 25 | - arm64 26 | x-stainless-async: 27 | - async:asyncio 28 | x-stainless-lang: 29 | - python 30 | x-stainless-os: 31 | - MacOS 32 | x-stainless-package-version: 33 | - 0.39.0 34 | x-stainless-retry-count: 35 | - '0' 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.11.10 40 | method: POST 41 | uri: https://api.anthropic.com/v1/messages 42 | response: 43 | body: 44 | string: '{"id":"msg_01EssLNatxqdSzkXEX1veU71","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''m 45 | doing well, thank you for asking! How are you today?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":13,"output_tokens":18}}' 46 | headers: 47 | CF-Cache-Status: 48 | - DYNAMIC 49 | CF-RAY: 50 | - 8ed754a62dbd3364-MIA 51 | Connection: 52 | - keep-alive 53 | Content-Type: 54 | - application/json 55 | Date: 56 | - Thu, 05 Dec 2024 22:04:13 GMT 57 | Server: 58 | - cloudflare 59 | Transfer-Encoding: 60 | - chunked 61 | X-Robots-Tag: 62 | - none 63 | anthropic-ratelimit-input-tokens-limit: 64 | - '400000' 65 | anthropic-ratelimit-input-tokens-remaining: 66 | - '400000' 67 | anthropic-ratelimit-input-tokens-reset: 68 | - '2024-12-05T22:04:13Z' 69 | anthropic-ratelimit-output-tokens-limit: 70 | - '80000' 71 | anthropic-ratelimit-output-tokens-remaining: 72 | - '80000' 73 | anthropic-ratelimit-output-tokens-reset: 74 | - '2024-12-05T22:04:13Z' 75 | anthropic-ratelimit-requests-limit: 76 | - '4000' 77 | anthropic-ratelimit-requests-remaining: 78 | - '3999' 79 | anthropic-ratelimit-requests-reset: 80 | - '2024-12-05T22:04:13Z' 81 | anthropic-ratelimit-tokens-limit: 82 | - '480000' 83 | anthropic-ratelimit-tokens-remaining: 84 | - '480000' 85 | anthropic-ratelimit-tokens-reset: 86 | - '2024-12-05T22:04:13Z' 87 | content-length: 88 | - '296' 89 | request-id: 90 | - req_01TvAUtsH6Hdsj1YJrdMKmov 91 | via: 92 | - 1.1 google 93 | status: 94 | code: 200 95 | message: OK 96 | version: 1 97 | -------------------------------------------------------------------------------- /tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_async[async-openai].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model": 4 | "gpt-4o-mini"}' 5 | headers: 6 | accept: 7 | - application/json 8 | accept-encoding: 9 | - gzip, deflate 10 | connection: 11 | - keep-alive 12 | content-length: 13 | - '90' 14 | content-type: 15 | - application/json 16 | host: 17 | - api.openai.com 18 | user-agent: 19 | - AsyncOpenAI/Python 1.55.0 20 | x-stainless-arch: 21 | - arm64 22 | x-stainless-async: 23 | - async:asyncio 24 | x-stainless-lang: 25 | - python 26 | x-stainless-os: 27 | - MacOS 28 | x-stainless-package-version: 29 | - 1.55.0 30 | x-stainless-retry-count: 31 | - '0' 32 | x-stainless-runtime: 33 | - CPython 34 | x-stainless-runtime-version: 35 | - 3.11.10 36 | method: POST 37 | uri: https://api.openai.com/v1/chat/completions 38 | response: 39 | body: 40 | string: "{\n \"id\": \"chatcmpl-AbEFAncbnpE6Od0bEuqJ2jnc1IqLo\",\n \"object\": 41 | \"chat.completion\",\n \"created\": 1733436252,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n 42 | \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": 43 | \"assistant\",\n \"content\": \"Hello! I'm just a program, but I'm 44 | here and ready to assist you. How can I help you today?\",\n \"refusal\": 45 | null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n 46 | \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 13,\n \"completion_tokens\": 47 | 23,\n \"total_tokens\": 36,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 48 | 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": 49 | {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 50 | 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"system_fingerprint\": 51 | \"fp_0705bf87c0\"\n}\n" 52 | headers: 53 | CF-Cache-Status: 54 | - DYNAMIC 55 | CF-RAY: 56 | - 8ed754a2299831dd-MIA 57 | Connection: 58 | - keep-alive 59 | Content-Type: 60 | - application/json 61 | Date: 62 | - Thu, 05 Dec 2024 22:04:13 GMT 63 | Server: 64 | - cloudflare 65 | Set-Cookie: 66 | - __cf_bm=nHZCNCDMfCHJqFbcd2_XkcnIcFbzHRbCkwTZ3gIwOcY-1733436253-1.0.1.1-RWOKr2iIFIYlKTEypVjPCRHSbSGYb537RmI8WyAkEOH6NndLax0K2FEev9tiJoBT9kBAjamGvQuzvkSA5jdkrw; 67 | path=/; expires=Thu, 05-Dec-24 22:34:13 GMT; domain=.api.openai.com; HttpOnly; 68 | Secure; SameSite=None 69 | - _cfuvid=vsNSK7Vpu5wJTa0Tc99hhlEkhz5sVR4OtPP6jpH4VRk-1733436253076-0.0.1.1-604800000; 70 | path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None 71 | Transfer-Encoding: 72 | - chunked 73 | X-Content-Type-Options: 74 | - nosniff 75 | access-control-expose-headers: 76 | - X-Request-ID 77 | alt-svc: 78 | - h3=":443"; ma=86400 79 | content-length: 80 | - '841' 81 | openai-organization: 82 | - not-diamond-bdz8cg 83 | openai-processing-ms: 84 | - '445' 85 | openai-version: 86 | - '2020-10-01' 87 | strict-transport-security: 88 | - max-age=31536000; includeSubDomains; preload 89 | x-ratelimit-limit-requests: 90 | - '30000' 91 | x-ratelimit-limit-tokens: 92 | - '150000000' 93 | x-ratelimit-remaining-requests: 94 | - '29999' 95 | x-ratelimit-remaining-tokens: 96 | - '149999978' 97 | x-ratelimit-reset-requests: 98 | - 2ms 99 | x-ratelimit-reset-tokens: 100 | - 0s 101 | x-request-id: 102 | - req_22ad18fdbfdbce34e3d5ed53a6ac609e 103 | status: 104 | code: 200 105 | message: OK 106 | version: 1 107 | -------------------------------------------------------------------------------- /tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_async_create[async-anthropic].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "Hello, how 4 | are you?"}], "model": "claude-3-5-haiku-20241022"}' 5 | headers: 6 | accept: 7 | - application/json 8 | accept-encoding: 9 | - gzip, deflate 10 | anthropic-version: 11 | - '2023-06-01' 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - '124' 16 | content-type: 17 | - application/json 18 | host: 19 | - api.anthropic.com 20 | user-agent: 21 | - AsyncAnthropic/Python 0.39.0 22 | x-api-key: 23 | - MASKED-API-KEY-FOR-TESTING 24 | x-stainless-arch: 25 | - arm64 26 | x-stainless-async: 27 | - async:asyncio 28 | x-stainless-lang: 29 | - python 30 | x-stainless-os: 31 | - MacOS 32 | x-stainless-package-version: 33 | - 0.39.0 34 | x-stainless-retry-count: 35 | - '0' 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.11.10 40 | method: POST 41 | uri: https://api.anthropic.com/v1/messages 42 | response: 43 | body: 44 | string: '{"id":"msg_01Rewjq1gNNLooVwwqFgrGc8","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''m 45 | doing well, thanks for asking! How are you today? Is there anything I can 46 | help you with?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":13,"output_tokens":26}}' 47 | headers: 48 | CF-Cache-Status: 49 | - DYNAMIC 50 | CF-RAY: 51 | - 8edf71fda9a5da9f-MIA 52 | Connection: 53 | - keep-alive 54 | Content-Type: 55 | - application/json 56 | Date: 57 | - Fri, 06 Dec 2024 21:42:21 GMT 58 | Server: 59 | - cloudflare 60 | Transfer-Encoding: 61 | - chunked 62 | X-Robots-Tag: 63 | - none 64 | anthropic-ratelimit-input-tokens-limit: 65 | - '400000' 66 | anthropic-ratelimit-input-tokens-remaining: 67 | - '400000' 68 | anthropic-ratelimit-input-tokens-reset: 69 | - '2024-12-06T21:42:21Z' 70 | anthropic-ratelimit-output-tokens-limit: 71 | - '80000' 72 | anthropic-ratelimit-output-tokens-remaining: 73 | - '80000' 74 | anthropic-ratelimit-output-tokens-reset: 75 | - '2024-12-06T21:42:21Z' 76 | anthropic-ratelimit-requests-limit: 77 | - '4000' 78 | anthropic-ratelimit-requests-remaining: 79 | - '3999' 80 | anthropic-ratelimit-requests-reset: 81 | - '2024-12-06T21:42:21Z' 82 | anthropic-ratelimit-tokens-limit: 83 | - '480000' 84 | anthropic-ratelimit-tokens-remaining: 85 | - '480000' 86 | anthropic-ratelimit-tokens-reset: 87 | - '2024-12-06T21:42:21Z' 88 | content-length: 89 | - '332' 90 | request-id: 91 | - req_01EeR56CGL4p7RCzSgbZiucu 92 | via: 93 | - 1.1 google 94 | status: 95 | code: 200 96 | message: OK 97 | version: 1 98 | -------------------------------------------------------------------------------- /tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_async_create[async-azure-openai].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model": 4 | "gpt-4o-mini"}' 5 | headers: 6 | accept: 7 | - application/json 8 | accept-encoding: 9 | - gzip, deflate, zstd 10 | api-key: 11 | - VkhQnoqnZjxaq2NSprIB6ElTip9DgcBAYYA9s8DieI0U2q8C61C0JQQJ99BBACYeBjFXJ3w3AAABACOGP4m6 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - '90' 16 | content-type: 17 | - application/json 18 | host: 19 | - notdiamond-azure-openai.openai.azure.com 20 | user-agent: 21 | - AsyncAzureOpenAI/Python 1.68.2 22 | x-stainless-arch: 23 | - x64 24 | x-stainless-async: 25 | - async:asyncio 26 | x-stainless-lang: 27 | - python 28 | x-stainless-os: 29 | - Linux 30 | x-stainless-package-version: 31 | - 1.68.2 32 | x-stainless-read-timeout: 33 | - '20.0' 34 | x-stainless-retry-count: 35 | - '0' 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.12.3 40 | method: POST 41 | uri: https://notdiamond-azure-openai.openai.azure.com/openai/deployments/gpt-4o-mini/chat/completions?api-version=2024-07-01-preview 42 | response: 43 | body: 44 | string: '{"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"Hello! 45 | I''m just a program, but I''m here and ready to help you. How can I assist 46 | you today?","refusal":null,"role":"assistant"}}],"created":1742891954,"id":"chatcmpl-BEu6UuKjF02da3tOs1IUoI1rzmNxm","model":"gpt-4o-mini-2024-07-18","object":"chat.completion","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}],"system_fingerprint":"fp_b705f0c291","usage":{"completion_tokens":24,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":13,"prompt_tokens_details":{"audio_tokens":0,"cached_tokens":0},"total_tokens":37}} 47 | 48 | ' 49 | headers: 50 | Content-Length: 51 | - '1112' 52 | Content-Type: 53 | - application/json 54 | Date: 55 | - Tue, 25 Mar 2025 08:39:14 GMT 56 | Strict-Transport-Security: 57 | - max-age=31536000; includeSubDomains; preload 58 | apim-request-id: 59 | - be2f6346-6c44-4a1a-ac24-23f8e17b4c37 60 | azureml-model-session: 61 | - v20250319-1-164616836 62 | cmp-upstream-response-duration: 63 | - '587' 64 | ms-azureml-model-time: 65 | - '635' 66 | x-accel-buffering: 67 | - 'no' 68 | x-aml-cluster: 69 | - hyena-westus3-01 70 | x-content-type-options: 71 | - nosniff 72 | x-envoy-upstream-service-time: 73 | - '638' 74 | x-ms-client-request-id: 75 | - be2f6346-6c44-4a1a-ac24-23f8e17b4c37 76 | x-ms-rai-invoked: 77 | - 'true' 78 | x-ms-region: 79 | - East US 80 | x-ratelimit-limit-requests: 81 | - '20000' 82 | x-ratelimit-limit-tokens: 83 | - '2000000' 84 | x-ratelimit-remaining-requests: 85 | - '19996' 86 | x-ratelimit-remaining-tokens: 87 | - '1995630' 88 | x-request-id: 89 | - 21402e2a-3ef5-421e-a4f2-b6f1acbab9d0 90 | status: 91 | code: 200 92 | message: OK 93 | version: 1 94 | -------------------------------------------------------------------------------- /tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_create[anthropic].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"max_tokens": 1024, "messages": [{"role": "user", "content": "Hello, how 4 | are you?"}], "model": "claude-3-5-haiku-20241022"}' 5 | headers: 6 | accept: 7 | - application/json 8 | accept-encoding: 9 | - gzip, deflate 10 | anthropic-version: 11 | - '2023-06-01' 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - '124' 16 | content-type: 17 | - application/json 18 | host: 19 | - api.anthropic.com 20 | user-agent: 21 | - Anthropic/Python 0.39.0 22 | x-api-key: 23 | - MASKED-API-KEY-FOR-TESTING 24 | x-stainless-arch: 25 | - arm64 26 | x-stainless-async: 27 | - 'false' 28 | x-stainless-lang: 29 | - python 30 | x-stainless-os: 31 | - MacOS 32 | x-stainless-package-version: 33 | - 0.39.0 34 | x-stainless-retry-count: 35 | - '0' 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.11.10 40 | method: POST 41 | uri: https://api.anthropic.com/v1/messages 42 | response: 43 | body: 44 | string: '{"id":"msg_01FZDC9VQuPxgjxWaurydxLV","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''m 45 | doing well, thank you for asking! How are you today? Is there anything I can 46 | help you with?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":13,"output_tokens":27}}' 47 | headers: 48 | CF-Cache-Status: 49 | - DYNAMIC 50 | CF-RAY: 51 | - 8edf71f64d1a09ce-MIA 52 | Connection: 53 | - keep-alive 54 | Content-Type: 55 | - application/json 56 | Date: 57 | - Fri, 06 Dec 2024 21:42:20 GMT 58 | Server: 59 | - cloudflare 60 | Transfer-Encoding: 61 | - chunked 62 | X-Robots-Tag: 63 | - none 64 | anthropic-ratelimit-input-tokens-limit: 65 | - '400000' 66 | anthropic-ratelimit-input-tokens-remaining: 67 | - '400000' 68 | anthropic-ratelimit-input-tokens-reset: 69 | - '2024-12-06T21:42:20Z' 70 | anthropic-ratelimit-output-tokens-limit: 71 | - '80000' 72 | anthropic-ratelimit-output-tokens-remaining: 73 | - '80000' 74 | anthropic-ratelimit-output-tokens-reset: 75 | - '2024-12-06T21:42:20Z' 76 | anthropic-ratelimit-requests-limit: 77 | - '4000' 78 | anthropic-ratelimit-requests-remaining: 79 | - '3999' 80 | anthropic-ratelimit-requests-reset: 81 | - '2024-12-06T21:42:19Z' 82 | anthropic-ratelimit-tokens-limit: 83 | - '480000' 84 | anthropic-ratelimit-tokens-remaining: 85 | - '480000' 86 | anthropic-ratelimit-tokens-reset: 87 | - '2024-12-06T21:42:20Z' 88 | content-length: 89 | - '335' 90 | request-id: 91 | - req_01TAPLdMRJWY6m8vJGbZCmCQ 92 | via: 93 | - 1.1 google 94 | status: 95 | code: 200 96 | message: OK 97 | version: 1 98 | -------------------------------------------------------------------------------- /tests/test_toolkit/cassettes/test_retry/test_retry_wrapper_create[azure-openai].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Hello, how are you?"}], "model": 4 | "gpt-4o-mini"}' 5 | headers: 6 | accept: 7 | - application/json 8 | accept-encoding: 9 | - gzip, deflate, zstd 10 | api-key: 11 | - VkhQnoqnZjxaq2NSprIB6ElTip9DgcBAYYA9s8DieI0U2q8C61C0JQQJ99BBACYeBjFXJ3w3AAABACOGP4m6 12 | connection: 13 | - keep-alive 14 | content-length: 15 | - '90' 16 | content-type: 17 | - application/json 18 | host: 19 | - notdiamond-azure-openai.openai.azure.com 20 | user-agent: 21 | - AzureOpenAI/Python 1.68.2 22 | x-stainless-arch: 23 | - x64 24 | x-stainless-async: 25 | - 'false' 26 | x-stainless-lang: 27 | - python 28 | x-stainless-os: 29 | - Linux 30 | x-stainless-package-version: 31 | - 1.68.2 32 | x-stainless-read-timeout: 33 | - '20.0' 34 | x-stainless-retry-count: 35 | - '0' 36 | x-stainless-runtime: 37 | - CPython 38 | x-stainless-runtime-version: 39 | - 3.12.3 40 | method: POST 41 | uri: https://notdiamond-azure-openai.openai.azure.com/openai/deployments/gpt-4o-mini/chat/completions?api-version=2024-07-01-preview 42 | response: 43 | body: 44 | string: '{"choices":[{"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}},"finish_reason":"stop","index":0,"logprobs":null,"message":{"content":"Hello! 45 | I''m just a computer program, so I don''t have feelings, but I''m here and 46 | ready to help you. How can I assist you today?","refusal":null,"role":"assistant"}}],"created":1742891952,"id":"chatcmpl-BEu6SSU7lz2f9yspHEGqKXFVAbiMP","model":"gpt-4o-mini-2024-07-18","object":"chat.completion","prompt_filter_results":[{"prompt_index":0,"content_filter_results":{"hate":{"filtered":false,"severity":"safe"},"self_harm":{"filtered":false,"severity":"safe"},"sexual":{"filtered":false,"severity":"safe"},"violence":{"filtered":false,"severity":"safe"}}}],"system_fingerprint":"fp_ded0d14823","usage":{"completion_tokens":31,"completion_tokens_details":{"accepted_prediction_tokens":0,"audio_tokens":0,"reasoning_tokens":0,"rejected_prediction_tokens":0},"prompt_tokens":13,"prompt_tokens_details":{"audio_tokens":0,"cached_tokens":0},"total_tokens":44}} 47 | 48 | ' 49 | headers: 50 | Content-Length: 51 | - '1147' 52 | Content-Type: 53 | - application/json 54 | Date: 55 | - Tue, 25 Mar 2025 08:39:12 GMT 56 | Strict-Transport-Security: 57 | - max-age=31536000; includeSubDomains; preload 58 | apim-request-id: 59 | - af3a9be3-f831-4142-b782-d63640aa0369 60 | azureml-model-session: 61 | - v20250319-1-164616836 62 | cmp-upstream-response-duration: 63 | - '512' 64 | ms-azureml-model-time: 65 | - '549' 66 | x-accel-buffering: 67 | - 'no' 68 | x-aml-cluster: 69 | - hyena-westus3-02 70 | x-content-type-options: 71 | - nosniff 72 | x-envoy-upstream-service-time: 73 | - '551' 74 | x-ms-client-request-id: 75 | - af3a9be3-f831-4142-b782-d63640aa0369 76 | x-ms-rai-invoked: 77 | - 'true' 78 | x-ms-region: 79 | - East US 80 | x-ratelimit-limit-requests: 81 | - '20000' 82 | x-ratelimit-limit-tokens: 83 | - '2000000' 84 | x-ratelimit-remaining-requests: 85 | - '19997' 86 | x-ratelimit-remaining-tokens: 87 | - '1996275' 88 | x-request-id: 89 | - 2fecd1e8-fcfa-439a-8754-b8caff264432 90 | status: 91 | code: 200 92 | message: OK 93 | version: 1 94 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[anthropic-claude-3-5-sonnet-20240620-langchain_anthropic.ChatAnthropic].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "anthropic", "model": "claude-3-5-sonnet-20240620", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '314' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=d1ba7bb2-1139-49a0, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13df5fa78eec-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:55 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - d1ba7bb2-1139-49a0 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[anthropic-claude-3-haiku-20240307-langchain_anthropic.ChatAnthropic].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "anthropic", "model": "claude-3-haiku-20240307", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '311' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=32e65e02-7163-476f, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13ebee865afa-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:57 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 32e65e02-7163-476f 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[anthropic-claude-3-opus-20240229-langchain_anthropic.ChatAnthropic].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "anthropic", "model": "claude-3-opus-20240229", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '310' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=8178b5b5-5d5d-4a3e, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13e46dd1c314-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:56 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 8178b5b5-5d5d-4a3e 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[anthropic-claude-3-sonnet-20240229-langchain_anthropic.ChatAnthropic].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "anthropic", "model": "claude-3-sonnet-20240229", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '312' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=18e74451-fdd1-4daa, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13e87909dfcc-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:57 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 18e74451-fdd1-4daa 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[cohere-command-r-langchain_cohere.ChatCohere].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "cohere", "model": "command-r", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '294' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=e8b23219-8ce1-474b, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d1435df6a5b78-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:09 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - e8b23219-8ce1-474b 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[cohere-command-r-plus-langchain_cohere.ChatCohere].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "cohere", "model": "command-r-plus", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '299' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=5ddfef53-5588-4818, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d14323a805b6c-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:08 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 5ddfef53-5588-4818 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[google-gemini-1.5-flash-latest-langchain_google_genai.ChatGoogleGenerativeAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "google", "model": "gemini-1.5-flash-latest", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '308' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=a64972ee-d870-493e, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13f6ea343250-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:59 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - a64972ee-d870-493e 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[google-gemini-1.5-pro-latest-langchain_google_genai.ChatGoogleGenerativeAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "google", "model": "gemini-1.5-pro-latest", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '306' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=30b056f5-1ab7-46ee, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13f17a973dc0-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:58 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 30b056f5-1ab7-46ee 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-codestral-latest-langchain_mistralai.ChatMistralAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "mistral", "model": "codestral-latest", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '302' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=1728d3df-69c9-4524, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13fc6e665a5f-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:00 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 1728d3df-69c9-4524 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-mistral-large-2402-langchain_mistralai.ChatMistralAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "mistral", "model": "mistral-large-2402", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '304' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=f3150336-ac6a-4f64, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d14056ef15bad-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:01 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - f3150336-ac6a-4f64 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-mistral-large-2407-langchain_mistralai.ChatMistralAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "mistral", "model": "mistral-large-2407", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '304' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=d5603514-a7fa-4a40, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d14029b91d0cb-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:01 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - d5603514-a7fa-4a40 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-mistral-medium-latest-langchain_mistralai.ChatMistralAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "mistral", "model": "mistral-medium-latest", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '307' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=953f08fc-c5c1-495e, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d1408c9379aa0-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:02 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 953f08fc-c5c1-495e 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-mistral-small-latest-langchain_mistralai.ChatMistralAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "mistral", "model": "mistral-small-latest", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '306' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=7aebc438-5a94-45b1, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d140b5d425ba3-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:02 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 7aebc438-5a94-45b1 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-open-mistral-7b-langchain_mistralai.ChatMistralAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "mistral", "model": "open-mistral-7b", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '301' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=cb026621-bf3c-487f, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d14107888d0ea-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:03 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - cb026621-bf3c-487f 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-open-mixtral-8x22b-langchain_mistralai.ChatMistralAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "mistral", "model": "open-mixtral-8x22b", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '304' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=1ce7a93b-eafe-46a8, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13f9aa335a9b-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:59 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 1ce7a93b-eafe-46a8 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[mistral-open-mixtral-8x7b-langchain_mistralai.ChatMistralAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "mistral", "model": "open-mixtral-8x7b", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '303' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=d4d6a167-30ab-4d2d, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13ffedafd0c4-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:00 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - d4d6a167-30ab-4d2d 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-3.5-turbo-0125-langchain_openai.ChatOpenAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "openai", "model": "gpt-3.5-turbo-0125", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '303' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=43551482-b870-4a26, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13d8bd658ef2-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:54 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 43551482-b870-4a26 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4-0125-preview-langchain_openai.ChatOpenAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "openai", "model": "gpt-4-0125-preview", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '303' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=deb88868-fc19-4780, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13cf4d41c26a-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:53 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - deb88868-fc19-4780 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4-0613-langchain_openai.ChatOpenAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "openai", "model": "gpt-4-0613", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '295' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=fe14af84-ff88-40c0, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13d58f97d0d7-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:54 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - fe14af84-ff88-40c0 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4-1106-preview-langchain_openai.ChatOpenAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "openai", "model": "gpt-4-1106-preview", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '303' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=7c0b1238-78f1-4d4c, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13d29b9e6e9e-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:53 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 7c0b1238-78f1-4d4c 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4-turbo-2024-04-09-langchain_openai.ChatOpenAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "openai", "model": "gpt-4-turbo-2024-04-09", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '307' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=47751cf0-39d4-466e, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13cc7d7589c3-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:52 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 47751cf0-39d4-466e 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4o-2024-05-13-langchain_openai.ChatOpenAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "openai", "model": "gpt-4o-2024-05-13", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '302' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=7089b607-7066-4c50, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13c92dedc2fa-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:52 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 7089b607-7066-4c50 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4o-2024-08-06-langchain_openai.ChatOpenAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "openai", "model": "gpt-4o-2024-08-06", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '302' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=6f3319f5-6e71-4aa4, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13c4f9d8c27d-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:51 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 6f3319f5-6e71-4aa4 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4o-langchain_openai.ChatOpenAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "openai", "model": "gpt-4o", "is_custom": false, "context_length": 5 | null, "input_price": null, "output_price": null, "latency": null}], "metric": 6 | "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '291' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=54dc61bb-1e8f-4a07, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13c25b982693-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:51 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 54dc61bb-1e8f-4a07 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[openai-gpt-4o-mini-2024-07-18-langchain_openai.ChatOpenAI].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "openai", "model": "gpt-4o-mini-2024-07-18", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '307' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=2e0bd7b2-8cc2-4377, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d13db6bb65bb2-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:35:55 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 2e0bd7b2-8cc2-4377 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Llama-3-70b-chat-hf-langchain_together.ChatTogether].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "togetherai", "model": "Llama-3-70b-chat-hf", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '308' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=dcb4a717-b0d0-4453, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d1412f91b1c23-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:03 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - dcb4a717-b0d0-4453 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Llama-3-8b-chat-hf-langchain_together.ChatTogether].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "togetherai", "model": "Llama-3-8b-chat-hf", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '307' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=fb0d20ad-5e6e-49fd, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d1415cdb125c0-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:04 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - fb0d20ad-5e6e-49fd 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Meta-Llama-3.1-405B-Instruct-Turbo-langchain_together.ChatTogether].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "togetherai", "model": "Meta-Llama-3.1-405B-Instruct-Turbo", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '323' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=4775345a-1f88-43b7, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d141f198e1c9c-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:05 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 4775345a-1f88-43b7 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Meta-Llama-3.1-70B-Instruct-Turbo-langchain_together.ChatTogether].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "togetherai", "model": "Meta-Llama-3.1-70B-Instruct-Turbo", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '322' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=a13267ad-cf8a-4f48, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d141ba8432bd8-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:05 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - a13267ad-cf8a-4f48 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Meta-Llama-3.1-8B-Instruct-Turbo-langchain_together.ChatTogether].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "togetherai", "model": "Meta-Llama-3.1-8B-Instruct-Turbo", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '321' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=f7821e12-47f1-4480, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d14193a602bd8-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:04 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - f7821e12-47f1-4480 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Mistral-7B-Instruct-v0.2-langchain_together.ChatTogether].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "togetherai", "model": "Mistral-7B-Instruct-v0.2", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '313' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=08eaf899-2575-47a7, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d142eb8055db9-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:08 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 08eaf899-2575-47a7 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Mixtral-8x22B-Instruct-v0.1-langchain_together.ChatTogether].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "togetherai", "model": "Mixtral-8x22B-Instruct-v0.1", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '316' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=b6a7ffcb-f95e-4c7c, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d14261cb8c297-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:06 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - b6a7ffcb-f95e-4c7c 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Mixtral-8x7B-Instruct-v0.1-langchain_together.ChatTogether].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "togetherai", "model": "Mixtral-8x7B-Instruct-v0.1", "is_custom": 5 | false, "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '315' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=606ee8fb-d82b-472b, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d142c09c41ca0-SOF 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:07 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 606ee8fb-d82b-472b 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/langchain/cassettes/test_integration/test_invokable[togetherai-Qwen2-72B-Instruct-langchain_together.ChatTogether].yaml: -------------------------------------------------------------------------------- 1 | interactions: 2 | - request: 3 | body: '{"messages": [{"role": "user", "content": "Test prompt"}], "llm_providers": 4 | [{"provider": "togetherai", "model": "Qwen2-72B-Instruct", "is_custom": false, 5 | "context_length": null, "input_price": null, "output_price": null, "latency": 6 | null}], "metric": "accuracy", "max_model_depth": 1, "hash_content": false}' 7 | headers: 8 | Accept: 9 | - '*/*' 10 | Accept-Encoding: 11 | - gzip, deflate, zstd 12 | Connection: 13 | - keep-alive 14 | Content-Length: 15 | - '307' 16 | User-Agent: 17 | - langchain-community/0.3.41 18 | content-type: 19 | - application/json 20 | method: POST 21 | uri: https://staging-api.notdiamond.ai/v2/modelRouter/modelSelect 22 | response: 23 | body: 24 | string: '{"detail":"requestID=793a0740-e1b6-44f3, Could not find valid API key"}' 25 | headers: 26 | CF-RAY: 27 | - 925d1421cb9b9796-VIE 28 | Connection: 29 | - keep-alive 30 | Content-Type: 31 | - application/json 32 | Date: 33 | - Tue, 25 Mar 2025 08:36:06 GMT 34 | Server: 35 | - cloudflare 36 | Transfer-Encoding: 37 | - chunked 38 | alt-svc: 39 | - h3=":443"; ma=86400 40 | cf-cache-status: 41 | - DYNAMIC 42 | rndr-id: 43 | - 793a0740-e1b6-44f3 44 | vary: 45 | - Accept-Encoding 46 | x-render-origin-server: 47 | - uvicorn 48 | status: 49 | code: 401 50 | message: Unauthorized 51 | version: 1 52 | -------------------------------------------------------------------------------- /tests/test_toolkit/rag/conftest.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import pytest 3 | from llama_index.core import SimpleDirectoryReader 4 | 5 | from notdiamond.llms.config import EmbeddingConfig, LLMConfig 6 | from notdiamond.toolkit.rag.evaluation_dataset import ( 7 | RAGEvaluationDataset, 8 | RAGSample, 9 | ) 10 | 11 | 12 | def format_prompt(user_input: str, retrieved_contexts: list[str]) -> str: 13 | context = "\n".join(retrieved_contexts) 14 | prompt = f""" 15 | Use the following context to answer the question. 16 | 17 | Context: {context} 18 | 19 | Question: {user_input} 20 | """ 21 | return prompt 22 | 23 | 24 | @pytest.fixture 25 | def user_input(): 26 | return "What's the capital of France?" 27 | 28 | 29 | @pytest.fixture 30 | def retrieved_contexts(): 31 | return ["Paris is the capital and most populous city of France."] 32 | 33 | 34 | @pytest.fixture 35 | def response(): 36 | return "The capital of France is Paris." 37 | 38 | 39 | @pytest.fixture 40 | def reference(): 41 | return "Paris" 42 | 43 | 44 | @pytest.fixture 45 | def gpt_4o(): 46 | return LLMConfig.from_string("openai/gpt-4o") 47 | 48 | 49 | @pytest.fixture 50 | def sonnet_3_5(): 51 | return LLMConfig.from_string("anthropic/claude-3-5-sonnet-20241022") 52 | 53 | 54 | @pytest.fixture 55 | def openai_embedding(): 56 | return EmbeddingConfig.from_string("openai/text-embedding-3-large") 57 | 58 | 59 | @pytest.fixture 60 | def dataset(gpt_4o, user_input, retrieved_contexts, response, reference): 61 | samples = [ 62 | RAGSample( 63 | user_input=user_input, 64 | retrieved_contexts=retrieved_contexts, 65 | response=response, 66 | reference=reference, 67 | generation_prompt=format_prompt(user_input, retrieved_contexts), 68 | generator_llm=str(gpt_4o), 69 | ), 70 | RAGSample( 71 | user_input=user_input, 72 | retrieved_contexts=retrieved_contexts, 73 | response=response, 74 | reference=reference, 75 | generation_prompt=format_prompt(user_input, retrieved_contexts), 76 | generator_llm=str(gpt_4o), 77 | ), 78 | ] 79 | return RAGEvaluationDataset(samples) 80 | 81 | 82 | @pytest.fixture 83 | def pandas_dataset( 84 | gpt_4o, user_input, retrieved_contexts, response, reference 85 | ): 86 | data = { 87 | "user_input": [user_input, user_input], 88 | "retrieved_contexts": [retrieved_contexts, retrieved_contexts], 89 | "response": [response, response], 90 | "reference": [reference, reference], 91 | "generation_prompt": [ 92 | format_prompt(user_input, retrieved_contexts), 93 | format_prompt(user_input, retrieved_contexts), 94 | ], 95 | "generator_llm": [str(gpt_4o), str(gpt_4o)], 96 | } 97 | df = pd.DataFrame(data=data) 98 | return df 99 | 100 | 101 | @pytest.fixture 102 | def llamaindex_documents(): 103 | loader = SimpleDirectoryReader( 104 | input_files=[ 105 | "tests/static/airbnb_tos.md", 106 | ] 107 | ) 108 | docs = loader.load_data() 109 | return docs 110 | 111 | 112 | @pytest.fixture 113 | def test_queries(): 114 | return ["summarize airbnb's ToS", "What are the cancellation policies?"] 115 | -------------------------------------------------------------------------------- /tests/test_toolkit/rag/test_data_gen.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import pytest 3 | from llama_index.core import SimpleDirectoryReader 4 | from ragas.embeddings import BaseRagasEmbeddings 5 | from ragas.llms import BaseRagasLLM 6 | 7 | from notdiamond.llms.config import EmbeddingConfig, LLMConfig 8 | from notdiamond.toolkit.rag.document_loaders import DirectoryLoader 9 | from notdiamond.toolkit.rag.llms import get_embedding, get_llm 10 | from notdiamond.toolkit.rag.testset import TestDataGenerator 11 | 12 | 13 | @pytest.fixture 14 | def test_data_langchain_docs(): 15 | loader = DirectoryLoader("tests/static/", glob="airbnb_tos.md") 16 | docs = loader.load() 17 | return docs 18 | 19 | 20 | @pytest.fixture 21 | def test_data_llamaindex_docs(): 22 | loader = SimpleDirectoryReader( 23 | input_files=[ 24 | "tests/static/airbnb_tos.md", 25 | ] 26 | ) 27 | docs = loader.load_data() 28 | return docs 29 | 30 | 31 | @pytest.fixture 32 | def generator_llm(): 33 | return get_llm("openai/gpt-4o-mini") 34 | 35 | 36 | @pytest.fixture 37 | def generator_embedding(): 38 | return get_embedding("openai/text-embedding-3-small") 39 | 40 | 41 | def test_get_llm_by_config(): 42 | llm_config = LLMConfig(provider="openai", model="gpt-4o-mini") 43 | llm = get_llm(llm_config) 44 | assert isinstance(llm, BaseRagasLLM) 45 | 46 | 47 | def test_get_embedding_by_config(): 48 | embed_config = EmbeddingConfig( 49 | provider="openai", model="text-embedding-3-small" 50 | ) 51 | embed = get_embedding(embed_config) 52 | assert isinstance(embed, BaseRagasEmbeddings) 53 | 54 | 55 | def test_dataset_generator_langchain_docs( 56 | test_data_langchain_docs, generator_llm, generator_embedding 57 | ): 58 | generator = TestDataGenerator( 59 | llm=generator_llm, embedding_model=generator_embedding 60 | ) 61 | dataset = generator.generate_from_docs( 62 | test_data_langchain_docs, testset_size=2 63 | ) 64 | assert len(dataset) == 2 65 | assert isinstance(dataset, pd.DataFrame) 66 | 67 | 68 | def test_dataset_generator_llamaindex_docs( 69 | test_data_llamaindex_docs, generator_llm, generator_embedding 70 | ): 71 | generator = TestDataGenerator( 72 | llm=generator_llm, embedding_model=generator_embedding 73 | ) 74 | dataset = generator.generate_from_docs( 75 | test_data_llamaindex_docs, testset_size=2 76 | ) 77 | assert len(dataset) == 2 78 | assert isinstance(dataset, pd.DataFrame) 79 | -------------------------------------------------------------------------------- /tests/test_toolkit/rag/test_evaluation.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from notdiamond.toolkit.rag.evaluation import evaluate 4 | from notdiamond.toolkit.rag.evaluation_dataset import ( 5 | RAGEvaluationDataset, 6 | RAGSample, 7 | ) 8 | from notdiamond.toolkit.rag.llms import get_embedding, get_llm 9 | from notdiamond.toolkit.rag.metrics import Faithfulness, SemanticSimilarity 10 | 11 | 12 | def test_dataset_from_pandas(pandas_dataset): 13 | dataset = RAGEvaluationDataset.from_pandas(pandas_dataset) 14 | assert isinstance(dataset, RAGEvaluationDataset) 15 | 16 | 17 | def test_dataset_format(dataset): 18 | assert len(dataset) == 2 19 | assert isinstance(dataset[0], RAGSample) 20 | assert isinstance(dataset[1], RAGSample) 21 | 22 | 23 | @pytest.mark.vcr 24 | def test_evaluate(dataset, gpt_4o, sonnet_3_5, openai_embedding): 25 | evaluator_llm = get_llm(gpt_4o) 26 | evaluator_embedding = get_embedding(openai_embedding) 27 | 28 | metrics = [ 29 | Faithfulness(llm=evaluator_llm), 30 | SemanticSimilarity(embeddings=evaluator_embedding), 31 | ] 32 | 33 | generator_llms = [gpt_4o, sonnet_3_5] 34 | 35 | results = evaluate( 36 | dataset=dataset, 37 | metrics=metrics, 38 | generator_llms=generator_llms, 39 | ) 40 | 41 | expected_results_keys = [str(gpt_4o), str(sonnet_3_5)] 42 | assert all([key in expected_results_keys for key in results.keys()]) 43 | 44 | gpt_4o_result = results[str(gpt_4o)] 45 | expected_results_columns = [ 46 | "user_input", 47 | "generation_prompt", 48 | "retrieved_contexts", 49 | "response", 50 | "reference", 51 | "faithfulness", 52 | "semantic_similarity", 53 | ] 54 | assert all( 55 | [ 56 | col in expected_results_columns 57 | for col in list(gpt_4o_result.columns) 58 | ] 59 | ) 60 | -------------------------------------------------------------------------------- /tests/test_toolkit/rag/test_workflow.py: -------------------------------------------------------------------------------- 1 | from typing import Annotated, Any, List 2 | 3 | import pytest 4 | 5 | from notdiamond.toolkit.rag.workflow import BaseNDRagWorkflow, IntValueRange 6 | 7 | 8 | class TestNDRagWorkflow(BaseNDRagWorkflow): 9 | parameter_specs = { 10 | "chunk_size": (Annotated[int, IntValueRange(1000, 2500, 500)], 1000) 11 | } 12 | 13 | def rag_workflow(self, documents: Any, test_queries: List[str]): 14 | pass 15 | 16 | 17 | def test_set_param_values(llamaindex_documents, test_queries): 18 | workflow = TestNDRagWorkflow( 19 | llamaindex_documents, test_queries, objective_maximize=True 20 | ) 21 | workflow._set_param_values({"chunk_size": 1500}) 22 | assert workflow.chunk_size == 1500 23 | 24 | with pytest.raises(ValueError): 25 | workflow._set_param_values({"nonexistent_param": 1500}) 26 | 27 | with pytest.raises(ValueError): 28 | workflow._set_param_values({"chunk_size": "not an int"}) 29 | 30 | # value too low for range 31 | with pytest.raises(ValueError): 32 | workflow._set_param_values({"chunk_size": 100}) 33 | 34 | 35 | def test_set_bad_param_values(test_queries, llamaindex_documents): 36 | class BadNDRagWorkflow(BaseNDRagWorkflow): 37 | parameter_specs = {"chunk_size": (int, 1000)} 38 | 39 | def rag_workflow(self, documents: Any, test_queries: List[str]): 40 | pass 41 | 42 | # should fail bc we need ranges 43 | with pytest.raises(ValueError): 44 | BadNDRagWorkflow( 45 | llamaindex_documents, test_queries, objective_maximize=True 46 | ) 47 | 48 | class BadNDRagWorkflow2(BaseNDRagWorkflow): 49 | parameter_specs = {"chunk_size": (int, "foo")} 50 | 51 | # should fail bc we need matching types 52 | with pytest.raises(ValueError): 53 | BadNDRagWorkflow2( 54 | llamaindex_documents, test_queries, objective_maximize=True 55 | ) 56 | -------------------------------------------------------------------------------- /tests/test_types.py: -------------------------------------------------------------------------------- 1 | """Tests for the types defined in the Python library""" 2 | 3 | 4 | import sys 5 | 6 | import pytest 7 | 8 | from notdiamond.exceptions import InvalidApiKey, MissingApiKey 9 | from notdiamond.types import NDApiKeyValidator 10 | 11 | sys.path.append("../") 12 | 13 | 14 | def test_api_key_not_string_fails(): 15 | """When the API key is not a string, the validation fails.""" 16 | with pytest.raises(InvalidApiKey): 17 | NDApiKeyValidator(api_key=1) 18 | 19 | with pytest.raises(InvalidApiKey): 20 | NDApiKeyValidator(api_key=None) 21 | 22 | 23 | def test_api_key_empty_string_fails(): 24 | """When the API key is an empty string, the validation fails.""" 25 | with pytest.raises(MissingApiKey): 26 | NDApiKeyValidator(api_key="") 27 | 28 | 29 | def test_api_key_passes(): 30 | """When the API key is a valid string and not empty, validation passes successfully.""" 31 | notdiamond_api_key = NDApiKeyValidator(api_key="NDAPIKEY") 32 | assert notdiamond_api_key.api_key == "NDAPIKEY" 33 | --------------------------------------------------------------------------------