├── .all-contributorsrc ├── .circleci ├── config.yml └── requirements.txt ├── .env.example ├── .gitattributes ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.yml │ ├── config.yml │ └── feature_request.yml └── workflows │ └── docker.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── cookbook ├── Benchmarking_LLMs_by_use_case.ipynb ├── Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb ├── Evaluating_LLMs.ipynb ├── LiteLLM_AB_TestLLMs.ipynb ├── LiteLLM_Azure_OpenAI.ipynb ├── LiteLLM_Azure_and_OpenAI_example.ipynb ├── LiteLLM_Bedrock.ipynb ├── LiteLLM_Caching.ipynb ├── LiteLLM_Comparing_LLMs.ipynb ├── LiteLLM_Completion_Cost.ipynb ├── LiteLLM_GPTCache.ipynb ├── LiteLLM_HuggingFace.ipynb ├── LiteLLM_OpenRouter.ipynb ├── LiteLLM_Petals.ipynb ├── LiteLLM_PromptLayer.ipynb ├── LiteLLM_User_Based_Rate_Limits.ipynb ├── LiteLLM_batch_completion.ipynb ├── TogetherAI_liteLLM.ipynb ├── VLLM_Model_Testing.ipynb ├── benchmark │ ├── benchmark.py │ └── readme.md ├── codellama-server │ ├── README.MD │ ├── imgs │ │ ├── code-output.png │ │ └── promptlayer_logging.png │ └── main.py ├── community-resources │ └── max_tokens.json ├── get_hf_models.py ├── liteLLM_A121_Jurrasic_example.ipynb ├── liteLLM_Baseten.ipynb ├── liteLLM_ChromaDB_Cache.ipynb ├── liteLLM_Getting_Started.ipynb ├── liteLLM_Langchain_Demo.ipynb ├── liteLLM_Ollama.ipynb ├── liteLLM_Replicate_Demo.ipynb ├── liteLLM_Streaming_Demo.ipynb ├── liteLLM_VertextAI_Example.ipynb ├── liteLLM_function_calling.ipynb ├── litellm-ollama-docker-image │ ├── Dockerfile │ ├── requirements.txt │ ├── start.sh │ └── test.py ├── litellm_Test_Multiple_Providers.ipynb ├── litellm_model_fallback.ipynb ├── litellm_test_multiple_llm_demo.ipynb ├── llm-ab-test-server │ ├── main.py │ └── readme.md ├── logging_observability │ └── LiteLLM_Langfuse.ipynb └── proxy-server │ └── readme.md ├── data_map.txt ├── dist ├── litellm-0.1.401-py3-none-any.whl ├── litellm-0.1.401.tar.gz ├── litellm-0.1.432-py3-none-any.whl ├── litellm-0.1.432.tar.gz ├── litellm-0.1.434-py3-none-any.whl ├── litellm-0.1.434.tar.gz ├── litellm-0.1.435-py3-none-any.whl ├── litellm-0.1.435.tar.gz ├── litellm-0.1.446-py3-none-any.whl ├── litellm-0.1.446.tar.gz ├── litellm-0.1.452-py3-none-any.whl ├── litellm-0.1.452.tar.gz ├── litellm-0.1.455-py3-none-any.whl ├── litellm-0.1.455.tar.gz ├── litellm-0.1.456-py3-none-any.whl ├── litellm-0.1.456.tar.gz ├── litellm-0.1.486-py3-none-any.whl ├── litellm-0.1.486.tar.gz ├── litellm-0.1.487-py3-none-any.whl ├── litellm-0.1.487.tar.gz ├── litellm-0.1.517-py3-none-any.whl ├── litellm-0.1.517.tar.gz ├── litellm-0.1.535-py3-none-any.whl ├── litellm-0.1.535.tar.gz ├── litellm-0.1.536-py3-none-any.whl ├── litellm-0.1.536.tar.gz ├── litellm-0.1.537-py3-none-any.whl ├── litellm-0.1.537.tar.gz ├── litellm-0.1.546-py3-none-any.whl ├── litellm-0.1.546.tar.gz ├── litellm-0.1.547-py3-none-any.whl ├── litellm-0.1.547.tar.gz ├── litellm-0.1.548-py3-none-any.whl ├── litellm-0.1.548.tar.gz ├── litellm-0.1.549-py3-none-any.whl ├── litellm-0.1.549.tar.gz ├── litellm-0.1.550-py3-none-any.whl ├── litellm-0.1.550.tar.gz ├── litellm-0.1.551-py3-none-any.whl ├── litellm-0.1.551.tar.gz ├── litellm-0.1.552-py3-none-any.whl ├── litellm-0.1.552.tar.gz ├── litellm-0.1.553-py3-none-any.whl ├── litellm-0.1.553.tar.gz ├── litellm-0.1.554-py3-none-any.whl ├── litellm-0.1.554.tar.gz ├── litellm-0.1.555-py3-none-any.whl ├── litellm-0.1.555.tar.gz ├── litellm-0.1.567-py3-none-any.whl ├── litellm-0.1.567.tar.gz ├── litellm-0.1.568-py3-none-any.whl ├── litellm-0.1.568.tar.gz ├── litellm-0.1.569-py3-none-any.whl ├── litellm-0.1.569.tar.gz ├── litellm-0.1.585-py3-none-any.whl ├── litellm-0.1.585.tar.gz ├── litellm-0.1.586-py3-none-any.whl ├── litellm-0.1.586.tar.gz ├── litellm-0.1.587-py3-none-any.whl ├── litellm-0.1.587.tar.gz ├── litellm-0.1.629-py3-none-any.whl ├── litellm-0.1.629.tar.gz ├── litellm-0.1.630-py3-none-any.whl ├── litellm-0.1.630.tar.gz ├── litellm-0.1.631-py3-none-any.whl ├── litellm-0.1.631.tar.gz ├── litellm-0.1.636-py3-none-any.whl ├── litellm-0.1.636.tar.gz ├── litellm-0.1.641-py3-none-any.whl ├── litellm-0.1.641.tar.gz ├── litellm-0.1.644-py3-none-any.whl ├── litellm-0.1.644.tar.gz ├── litellm-0.1.645-py3-none-any.whl ├── litellm-0.1.645.tar.gz ├── litellm-0.1.646-py3-none-any.whl ├── litellm-0.1.646.tar.gz ├── litellm-0.1.647-py3-none-any.whl ├── litellm-0.1.647.tar.gz ├── litellm-0.1.648-py3-none-any.whl ├── litellm-0.1.648.tar.gz ├── litellm-0.1.649-py3-none-any.whl ├── litellm-0.1.649.tar.gz ├── litellm-0.1.650-py3-none-any.whl ├── litellm-0.1.650.tar.gz ├── litellm-0.1.651.tar.gz ├── litellm-0.1.652-py3-none-any.whl ├── litellm-0.1.652.tar.gz ├── litellm-0.1.689-py3-none-any.whl ├── litellm-0.1.689.tar.gz ├── litellm-0.1.696-py3-none-any.whl ├── litellm-0.1.696.tar.gz ├── litellm-0.1.746-py3-none-any.whl ├── litellm-0.1.746.tar.gz ├── litellm-0.1.750-py3-none-any.whl ├── litellm-0.1.750.tar.gz ├── litellm-0.1.751-py3-none-any.whl ├── litellm-0.1.751.tar.gz ├── litellm-0.1.766-py3-none-any.whl ├── litellm-0.1.766.tar.gz ├── litellm-0.1.767-py3-none-any.whl ├── litellm-0.1.767.tar.gz ├── litellm-0.1.768-py3-none-any.whl ├── litellm-0.1.768.tar.gz ├── litellm-0.1.7681-py3-none-any.whl ├── litellm-0.1.7681.tar.gz ├── litellm-0.1.7682-py3-none-any.whl ├── litellm-0.1.7682.tar.gz ├── litellm-0.1.7701-py3-none-any.whl ├── litellm-0.1.7701.tar.gz ├── litellm-0.1.7712-py3-none-any.whl ├── litellm-0.1.7712.tar.gz ├── litellm-0.1.7713-py3-none-any.whl ├── litellm-0.1.7713.tar.gz ├── litellm-0.1.781-py3-none-any.whl ├── litellm-0.1.781.tar.gz ├── litellm-0.7.1.dev1-py3-none-any.whl ├── litellm-0.7.1.dev1.tar.gz ├── litellm-0.7.1.dev2-py3-none-any.whl ├── litellm-0.7.1.dev2.tar.gz ├── litellm-0.7.1.dev3-py3-none-any.whl ├── litellm-0.7.1.dev3.tar.gz ├── litellm-0.9.2.dev1-py3-none-any.whl └── litellm-0.9.2.dev1.tar.gz ├── docs └── my-website │ ├── .gitignore │ ├── README.md │ ├── babel.config.js │ ├── blog │ └── 2021-08-26-welcome │ │ └── index.md │ ├── docs │ ├── budget_manager.md │ ├── caching │ │ ├── caching_api.md │ │ ├── gpt_cache.md │ │ ├── local_caching.md │ │ └── redis_cache.md │ ├── completion │ │ ├── batching.md │ │ ├── config.md │ │ ├── function_call.md │ │ ├── input.md │ │ ├── message_trimming.md │ │ ├── mock_requests.md │ │ ├── model_alias.md │ │ ├── multiple_deployments.md │ │ ├── output.md │ │ ├── prompt_formatting.md │ │ ├── reliable_completions.md │ │ ├── stream.md │ │ └── token_usage.md │ ├── contact.md │ ├── debugging │ │ ├── hosted_debugging.md │ │ └── local_debugging.md │ ├── default_code_snippet.md │ ├── embedding │ │ ├── moderation.md │ │ └── supported_embedding.md │ ├── exception_mapping.md │ ├── extras │ │ └── contributing.md │ ├── getting_started.md │ ├── index.md │ ├── langchain │ │ └── langchain.md │ ├── migration.md │ ├── observability │ │ ├── callbacks.md │ │ ├── custom_callback.md │ │ ├── helicone_integration.md │ │ ├── langfuse_integration.md │ │ ├── langsmith_integration.md │ │ ├── llmonitor_integration.md │ │ ├── promptlayer_integration.md │ │ ├── sentry.md │ │ ├── slack_integration.md │ │ ├── supabase_integration.md │ │ ├── telemetry.md │ │ ├── traceloop_integration.md │ │ └── wandb_integration.md │ ├── projects.md │ ├── projects │ │ ├── Codium PR Agent.md │ │ ├── FastREPL.md │ │ ├── GPT Migrate.md │ │ ├── Langstream.md │ │ ├── LiteLLM Proxy.md │ │ ├── OpenInterpreter.md │ │ ├── Otter.md │ │ ├── PROMPTMETHEUS.md │ │ ├── Prompt2Model.md │ │ ├── Quivr.md │ │ ├── SalesGPT.md │ │ └── YiVal.md │ ├── providers │ │ ├── ai21.md │ │ ├── aleph_alpha.md │ │ ├── anthropic.md │ │ ├── aws_sagemaker.md │ │ ├── azure.md │ │ ├── baseten.md │ │ ├── bedrock.md │ │ ├── cohere.md │ │ ├── custom.md │ │ ├── custom_openai_proxy.md │ │ ├── deepinfra.md │ │ ├── huggingface.md │ │ ├── nlp_cloud.md │ │ ├── ollama.md │ │ ├── openai.md │ │ ├── openrouter.md │ │ ├── palm.md │ │ ├── perplexity.md │ │ ├── petals.md │ │ ├── replicate.md │ │ ├── togetherai.md │ │ ├── vertex.md │ │ └── vllm.md │ ├── proxy_api.md │ ├── proxy_server.md │ ├── rate_limit_manager.md │ ├── routing.md │ ├── set_keys.md │ ├── simple_proxy.md │ ├── troubleshoot.md │ └── tutorials │ │ ├── TogetherAI_liteLLM.md │ │ ├── ab_test_llms.md │ │ ├── azure_openai.md │ │ ├── compare_llms.md │ │ ├── compare_llms_2.md │ │ ├── fallbacks.md │ │ ├── finetuned_chat_gpt.md │ │ ├── first_playground.md │ │ ├── gradio_integration.md │ │ ├── huggingface_codellama.md │ │ ├── huggingface_tutorial.md │ │ ├── installation.md │ │ ├── litellm_Test_Multiple_Providers.md │ │ ├── litellm_Test_Multiple_Providers_files │ │ ├── litellm_Test_Multiple_Providers_11_0.png │ │ └── litellm_Test_Multiple_Providers_14_0.png │ │ ├── mock_completion.md │ │ ├── model_config_proxy.md │ │ ├── model_fallbacks.md │ │ ├── oobabooga.md │ │ ├── provider_specific_params.md │ │ ├── sagemaker_llms.md │ │ └── text_completion.md │ ├── docusaurus.config.js │ ├── img │ ├── ab_test_code.png │ ├── ab_test_logs.png │ ├── add_model.png │ ├── alt_dashboard.png │ ├── bench_llm.png │ ├── cloud_run0.png │ ├── cloud_run1.png │ ├── cloud_run2.png │ ├── cloud_run3.png │ ├── codellama_formatted_input.png │ ├── codellama_input.png │ ├── compare_llms.png │ ├── dash_output.png │ ├── dashboard.png │ ├── dashboard_log.png │ ├── dashboard_log_row.png │ ├── favicon.png │ ├── hf_inference_endpoint.png │ ├── hosted_debugger_usage_page.png │ ├── langfuse.png │ ├── langsmith.png │ ├── lite_logs.png │ ├── lite_logs2.png │ ├── litellm_streamlit_playground.png │ ├── promptlayer.png │ ├── sagemaker_deploy.png │ ├── sagemaker_domain.png │ ├── sagemaker_endpoint.png │ ├── sagemaker_jumpstart.png │ ├── slack.png │ ├── swagger.png │ ├── term_output.png │ ├── test_python_server_1.png │ ├── test_python_server_2.png │ ├── traceloop_dash.png │ └── wandb.png │ ├── index.md │ ├── package-lock.json │ ├── package.json │ ├── sidebars.js │ ├── src │ ├── components │ │ ├── CrispChat.js │ │ ├── HomepageFeatures │ │ │ ├── index.js │ │ │ └── styles.module.css │ │ ├── QuickStart.js │ │ ├── TokenGen.js │ │ ├── queryParamReader.js │ │ └── queryParamToken.js │ ├── css │ │ └── custom.css │ ├── pages-fake │ │ ├── index.js │ │ ├── index.module.css │ │ └── markdown-page.md │ └── pages │ │ ├── completion │ │ ├── input.md │ │ ├── output.md │ │ └── supported.md │ │ ├── contact.md │ │ ├── contributing.md │ │ ├── embedding │ │ └── supported_embedding.md │ │ ├── index.md │ │ ├── intro.md │ │ ├── observability │ │ ├── callbacks.md │ │ ├── helicone_integration.md │ │ └── supabase_integration.md │ │ ├── secret.md │ │ ├── stream.md │ │ ├── token_usage.md │ │ ├── troubleshoot.md │ │ ├── tutorial-basics │ │ ├── _category_.json │ │ ├── congratulations.md │ │ ├── create-a-blog-post.md │ │ ├── create-a-document.md │ │ ├── create-a-page.md │ │ ├── deploy-your-site.md │ │ └── markdown-features.mdx │ │ └── tutorial-extras │ │ ├── _category_.json │ │ ├── img │ │ ├── docsVersionDropdown.png │ │ └── localeDropdown.png │ │ ├── manage-docs-versions.md │ │ └── translate-your-site.md │ ├── static │ ├── .nojekyll │ └── img │ │ ├── docusaurus-social-card.png │ │ ├── docusaurus.png │ │ ├── favicon.ico │ │ ├── logo.svg │ │ ├── undraw_docusaurus_mountain.svg │ │ ├── undraw_docusaurus_react.svg │ │ └── undraw_docusaurus_tree.svg │ └── yarn.lock ├── litellm ├── .env.template ├── __init__.py ├── _version.py ├── budget_manager.py ├── caching.py ├── config.json ├── cost.json ├── exceptions.py ├── gpt_cache.py ├── integrations │ ├── __init__.py │ ├── aispend.py │ ├── berrispend.py │ ├── custom_logger.py │ ├── helicone.py │ ├── langfuse.py │ ├── langsmith.py │ ├── litedebugger.py │ ├── llmonitor.py │ ├── prompt_layer.py │ ├── supabase.py │ ├── traceloop.py │ └── weights_biases.py ├── llms │ ├── __init__.py │ ├── ai21.py │ ├── aleph_alpha.py │ ├── anthropic.py │ ├── base.py │ ├── baseten.py │ ├── bedrock.py │ ├── cohere.py │ ├── huggingface_llms_metadata │ │ ├── hf_conversational_models.txt │ │ └── hf_text_generation_models.txt │ ├── huggingface_restapi.py │ ├── nlp_cloud.py │ ├── ollama.py │ ├── oobabooga.py │ ├── openai.py │ ├── palm.py │ ├── petals.py │ ├── prompt_templates │ │ └── factory.py │ ├── replicate.py │ ├── sagemaker.py │ ├── together_ai.py │ ├── tokenizers │ │ └── anthropic_tokenizer.json │ ├── vertex_ai.py │ └── vllm.py ├── main.py ├── model_prices_and_context_window_backup.json ├── proxy │ ├── .gitignore │ ├── README.md │ ├── __init__.py │ ├── api_log.json │ ├── cost.log │ ├── costs.json │ ├── llm.py │ ├── openapi.json │ ├── proxy_cli.py │ ├── proxy_server.py │ └── start.sh ├── router.py ├── testing.py ├── tests │ ├── api_log.json │ ├── data_map.txt │ ├── litellm_results.jsonl │ ├── litellm_uuid.txt │ ├── model_cost.json │ ├── test_add_function_to_prompt.py │ ├── test_api_key_param.py │ ├── test_async_fn.py │ ├── test_bad_params.py │ ├── test_batch_completions.py │ ├── test_budget_manager.py │ ├── test_caching.py │ ├── test_client.py │ ├── test_completion.py │ ├── test_completion_with_retries.py │ ├── test_config.py │ ├── test_custom_logger.py │ ├── test_embedding.py │ ├── test_exceptions.py │ ├── test_get_model_cost_map.py │ ├── test_get_model_file.py │ ├── test_get_model_list.py │ ├── test_helicone_integration.py │ ├── test_hf_prompt_templates.py │ ├── test_langchain_ChatLiteLLM.py │ ├── test_langfuse.py │ ├── test_langsmith.py │ ├── test_litedebugger_integration.py │ ├── test_litellm_max_budget.py │ ├── test_llmonitor_integration.py │ ├── test_load_test_model.py │ ├── test_logging.py │ ├── test_longer_context_fallback.py │ ├── test_mock_request.py │ ├── test_model_alias_map.py │ ├── test_model_fallback.py │ ├── test_model_response_typing │ │ ├── server.py │ │ └── test.py │ ├── test_multiple_deployments.py │ ├── test_no_client.py │ ├── test_ollama.py │ ├── test_ollama_local.py │ ├── test_prompt_factory.py │ ├── test_promptlayer_integration.py │ ├── test_provider_specific_config.py │ ├── test_proxy_api.py │ ├── test_rate_limit_manager.py │ ├── test_register_model.py │ ├── test_router.py │ ├── test_stream_chunk_builder.py │ ├── test_streaming.py │ ├── test_supabase_integration.py │ ├── test_timeout.py │ ├── test_token_counter.py │ ├── test_utils.py │ ├── test_validate_environment.py │ ├── test_vertex.py │ ├── test_wandb_logger.py │ └── user_cost.json ├── timeout.py └── utils.py ├── litellm_results.jsonl ├── model_prices_and_context_window.json ├── openai-proxy ├── .env.template ├── Dockerfile ├── README.md ├── main.py ├── requirements.txt └── tests │ ├── test_bedrock.py │ ├── test_openai.py │ └── test_openrouter.py ├── poetry.lock ├── pyproject.toml ├── render.yaml ├── requirements.txt └── secrets_template.toml /.all-contributorsrc: -------------------------------------------------------------------------------- 1 | { 2 | "projectName": "litellm", 3 | "projectOwner": "BerriAI" 4 | } 5 | -------------------------------------------------------------------------------- /.circleci/requirements.txt: -------------------------------------------------------------------------------- 1 | # used by CI/CD testing 2 | openai 3 | python-dotenv 4 | openai 5 | tiktoken 6 | importlib_metadata 7 | baseten 8 | cohere 9 | redis 10 | replicate 11 | anthropic 12 | gptcache 13 | boto3 14 | appdirs -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # OpenAI 2 | OPENAI_API_KEY = "" 3 | OPENAI_API_BASE = "" 4 | # Cohere 5 | COHERE_API_KEY = "" 6 | # OpenRouter 7 | OR_SITE_URL = "" 8 | OR_APP_NAME = "LiteLLM Example app" 9 | OR_API_KEY = "" 10 | # Azure API base URL 11 | AZURE_API_BASE = "" 12 | # Azure API version 13 | AZURE_API_VERSION = "" 14 | # Azure API key 15 | AZURE_API_KEY = "" 16 | # Replicate 17 | REPLICATE_API_KEY = "" 18 | REPLICATE_API_TOKEN = "" 19 | # Anthropic 20 | ANTHROPIC_API_KEY = "" 21 | # Infisical 22 | INFISICAL_TOKEN = "" 23 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.ipynb linguist-vendored -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | custom: https://buy.stripe.com/9AQ03Kd3P91o0Q8bIS 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: File a bug report 3 | title: "[Bug]: " 4 | labels: ["bug"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to fill out this bug report! 10 | - type: textarea 11 | id: what-happened 12 | attributes: 13 | label: What happened? 14 | description: Also tell us, what did you expect to happen? 15 | placeholder: Tell us what you see! 16 | value: "A bug happened!" 17 | validations: 18 | required: true 19 | - type: textarea 20 | id: logs 21 | attributes: 22 | label: Relevant log output 23 | description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. 24 | render: shell 25 | - type: input 26 | id: contact 27 | attributes: 28 | label: Twitter / LinkedIn details 29 | description: We announce new features on Twitter + LinkedIn. If this issue leads to an announcement, and you'd like a mention, we'll gladly shout you out! 30 | placeholder: ex. @krrish_dh / https://www.linkedin.com/in/krish-d/ 31 | validations: 32 | required: false 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Schedule Demo 4 | url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat 5 | about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM 6 | - name: Discord 7 | url: https://discord.com/invite/wuPM9dRgDw 8 | about: Join 250+ LiteLLM community members! 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yml: -------------------------------------------------------------------------------- 1 | name: 🚀 Feature Request 2 | description: Submit a proposal/request for a new LiteLLM feature. 3 | title: "[Feature]: " 4 | labels: ["enhancement"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for making LiteLLM better! 10 | - type: textarea 11 | id: the-feature 12 | attributes: 13 | label: The Feature 14 | description: A clear and concise description of the feature proposal 15 | placeholder: Tell us what you want! 16 | validations: 17 | required: true 18 | - type: textarea 19 | id: motivation 20 | attributes: 21 | label: Motivation, pitch 22 | description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too. 23 | validations: 24 | required: true 25 | - type: input 26 | id: contact 27 | attributes: 28 | label: Twitter / LinkedIn details 29 | description: We announce new features on Twitter + LinkedIn. When this is announced, and you'd like a mention, we'll gladly shout you out! 30 | placeholder: ex. @krrish_dh / https://www.linkedin.com/in/krish-d/ 31 | validations: 32 | required: false 33 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Build Docker Images 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | tag: 6 | description: "The tag version you want to build" 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@v4 13 | - name: Set up QEMU 14 | uses: docker/setup-qemu-action@v3 15 | - name: Set up Docker Buildx 16 | uses: docker/setup-buildx-action@v3 17 | - name: Login to GitHub Container Registry 18 | uses: docker/login-action@v3 19 | with: 20 | registry: ghcr.io 21 | username: ${{ github.actor }} 22 | password: ${{ secrets.GITHUB_TOKEN }} 23 | - name: Extract metadata (tags, labels) for Docker 24 | id: meta 25 | uses: docker/metadata-action@v5 26 | with: 27 | images: ghcr.io/${{ github.repository }} 28 | - name: Get tag to build 29 | id: tag 30 | run: | 31 | echo "latest=ghcr.io/${{ github.repository }}:latest" >> $GITHUB_OUTPUT 32 | if [[ -z "${{ github.event.inputs.tag }}" ]]; then 33 | echo "versioned=ghcr.io/${{ github.repository }}:${{ github.ref_name }}" >> $GITHUB_OUTPUT 34 | else 35 | echo "versioned=ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}" >> $GITHUB_OUTPUT 36 | fi 37 | - name: Build and release Docker images 38 | uses: docker/build-push-action@v5 39 | with: 40 | context: . 41 | platforms: linux/amd64,linux/arm64 42 | tags: | 43 | ${{ steps.tag.outputs.latest }} 44 | ${{ steps.tag.outputs.versioned }} 45 | labels: ${{ steps.meta.outputs.labels }} 46 | push: true -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | .env 3 | litellm_uuid.txt 4 | __pycache__/ 5 | *.pyc 6 | bun.lockb 7 | **/.DS_Store 8 | .aider* 9 | litellm_results.jsonl 10 | secrets.toml 11 | .gitignore 12 | litellm/proxy/litellm_secrets.toml 13 | litellm/proxy/api_log.json 14 | .idea/ -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | ENV LITELLM_CONFIG_PATH="/litellm.secrets.toml" 4 | COPY . /app 5 | WORKDIR /app 6 | RUN pip install -r requirements.txt 7 | 8 | WORKDIR /app/openai-proxy 9 | EXPOSE $PORT 10 | 11 | CMD exec uvicorn main:app --host 0.0.0.0 --port $PORT -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Berri AI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cookbook/codellama-server/imgs/code-output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/cookbook/codellama-server/imgs/code-output.png -------------------------------------------------------------------------------- /cookbook/codellama-server/imgs/promptlayer_logging.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/cookbook/codellama-server/imgs/promptlayer_logging.png -------------------------------------------------------------------------------- /cookbook/litellm-ollama-docker-image/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ollama/ollama as ollama 2 | 3 | RUN echo "auto installing llama2" 4 | 5 | # auto install ollama/llama2 6 | RUN ollama serve & sleep 2 && ollama pull llama2 7 | 8 | RUN echo "installing litellm" 9 | 10 | RUN apt-get update 11 | 12 | # Install Python 13 | RUN apt-get install -y python3 python3-pip 14 | 15 | # Set the working directory in the container 16 | WORKDIR /app 17 | 18 | # Copy the current directory contents into the container at /app 19 | COPY . /app 20 | 21 | # Install any needed packages specified in requirements.txt 22 | 23 | RUN python3 -m pip install litellm 24 | COPY start.sh /start.sh 25 | ENTRYPOINT [ "/bin/bash", "/start.sh" ] 26 | -------------------------------------------------------------------------------- /cookbook/litellm-ollama-docker-image/requirements.txt: -------------------------------------------------------------------------------- 1 | litellm -------------------------------------------------------------------------------- /cookbook/litellm-ollama-docker-image/start.sh: -------------------------------------------------------------------------------- 1 | ollama serve & 2 | litellm -------------------------------------------------------------------------------- /cookbook/litellm-ollama-docker-image/test.py: -------------------------------------------------------------------------------- 1 | 2 | import openai 3 | 4 | api_base = f"http://0.0.0.0:8000" 5 | 6 | openai.api_base = api_base 7 | openai.api_key = "temp-key" 8 | print(openai.api_base) 9 | 10 | 11 | print(f'LiteLLM: response from proxy with streaming') 12 | response = openai.ChatCompletion.create( 13 | model="ollama/llama2", 14 | messages = [ 15 | { 16 | "role": "user", 17 | "content": "this is a test request, acknowledge that you got it" 18 | } 19 | ], 20 | stream=True 21 | ) 22 | 23 | for chunk in response: 24 | print(f'LiteLLM: streaming response from proxy {chunk}') 25 | 26 | response = openai.ChatCompletion.create( 27 | model="ollama/llama2", 28 | messages = [ 29 | { 30 | "role": "user", 31 | "content": "this is a test request, acknowledge that you got it" 32 | } 33 | ] 34 | ) 35 | 36 | print(f'LiteLLM: response from proxy {response}') 37 | -------------------------------------------------------------------------------- /cookbook/litellm_model_fallback.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [] 7 | }, 8 | "kernelspec": { 9 | "name": "python3", 10 | "display_name": "Python 3" 11 | }, 12 | "language_info": { 13 | "name": "python" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "source": [ 20 | "!pip install litellm" 21 | ], 22 | "metadata": { 23 | "id": "j6yJsCGeaq8G" 24 | }, 25 | "execution_count": null, 26 | "outputs": [] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": null, 31 | "metadata": { 32 | "id": "u129iWNPaf72" 33 | }, 34 | "outputs": [], 35 | "source": [ 36 | "import litellm\n", 37 | "from litellm import embedding, completion\n", 38 | "\n", 39 | "model_fallback_list = [\"claude-instant-1\", \"gpt-3.5-turbo\", \"chatgpt-test\"]\n", 40 | "\n", 41 | "user_message = \"Hello, how are you?\"\n", 42 | "messages = [{ \"content\": user_message,\"role\": \"user\"}]\n", 43 | "\n", 44 | "for model in model_fallback_list:\n", 45 | " try:\n", 46 | " response = completion(model=model, messages=messages)\n", 47 | " except Exception as e:\n", 48 | " print(f\"error occurred: {traceback.format_exc()}\")" 49 | ] 50 | } 51 | ] 52 | } 53 | -------------------------------------------------------------------------------- /cookbook/litellm_test_multiple_llm_demo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [] 7 | }, 8 | "kernelspec": { 9 | "name": "python3", 10 | "display_name": "Python 3" 11 | }, 12 | "language_info": { 13 | "name": "python" 14 | } 15 | }, 16 | "cells": [ 17 | { 18 | "cell_type": "code", 19 | "execution_count": null, 20 | "metadata": { 21 | "id": "adotBkqZSh5g" 22 | }, 23 | "outputs": [], 24 | "source": [ 25 | "!pip install litellm" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "source": [ 31 | "from litellm import completion\n", 32 | "\n", 33 | "## set ENV variables\n", 34 | "os.environ[\"OPENAI_API_KEY\"] = \"openai key\"\n", 35 | "os.environ[\"COHERE_API_KEY\"] = \"cohere key\"\n", 36 | "os.environ[\"REPLICATE_API_KEY\"] = \"replicate key\"\n", 37 | "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", 38 | "\n", 39 | "# openai call\n", 40 | "response = completion(model=\"gpt-3.5-turbo\", messages=messages)\n", 41 | "\n", 42 | "# cohere call\n", 43 | "response = completion(\"command-nightly\", messages)\n", 44 | "\n", 45 | "# replicate call\n", 46 | "response = completion(\"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\", messages)" 47 | ], 48 | "metadata": { 49 | "id": "LeOqznSgSj-z" 50 | }, 51 | "execution_count": null, 52 | "outputs": [] 53 | } 54 | ] 55 | } 56 | -------------------------------------------------------------------------------- /cookbook/llm-ab-test-server/main.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, request, jsonify, abort, Response 2 | from flask_cors import CORS 3 | from litellm import completion 4 | import os, dotenv 5 | import random 6 | dotenv.load_dotenv() 7 | 8 | # TODO: set your keys in .env or here: 9 | # os.environ["OPENAI_API_KEY"] = "" # set your openai key here or in your .env 10 | # see supported models, keys here: 11 | 12 | 13 | app = Flask(__name__) 14 | CORS(app) 15 | 16 | @app.route('/') 17 | def index(): 18 | return 'received!', 200 19 | 20 | # Dictionary of LLM functions with their A/B test ratios, should sum to 1 :) 21 | llm_dict = { 22 | "gpt-4": 0.2, 23 | "together_ai/togethercomputer/llama-2-70b-chat": 0.4, 24 | "claude-2": 0.2, 25 | "claude-1.2": 0.2 26 | } 27 | 28 | 29 | @app.route('/chat/completions', methods=["POST"]) 30 | def api_completion(): 31 | data = request.json 32 | try: 33 | # pass in data to completion function, unpack data 34 | selected_llm = random.choices(list(llm_dict.keys()), weights=list(llm_dict.values()))[0] 35 | response = completion(**data, model=selected_llm) 36 | except Exception as e: 37 | print(f"got error{e}") 38 | return response, 200 39 | 40 | 41 | if __name__ == "__main__": 42 | from waitress import serve 43 | print("starting server") 44 | serve(app, host="0.0.0.0", port=5000, threads=500) 45 | 46 | -------------------------------------------------------------------------------- /data_map.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/data_map.txt -------------------------------------------------------------------------------- /dist/litellm-0.1.401-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.401-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.401.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.401.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.432-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.432-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.432.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.432.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.434-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.434-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.434.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.434.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.435-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.435-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.435.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.435.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.446-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.446-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.446.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.446.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.452-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.452-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.452.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.452.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.455-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.455-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.455.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.455.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.456-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.456-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.456.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.456.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.486-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.486-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.486.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.486.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.487-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.487-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.487.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.487.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.517-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.517-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.517.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.517.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.535-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.535-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.535.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.535.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.536-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.536-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.536.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.536.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.537-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.537-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.537.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.537.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.546-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.546-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.546.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.546.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.547-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.547-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.547.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.547.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.548-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.548-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.548.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.548.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.549-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.549-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.549.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.549.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.550-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.550-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.550.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.550.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.551-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.551-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.551.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.551.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.552-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.552-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.552.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.552.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.553-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.553-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.553.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.553.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.554-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.554-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.554.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.554.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.555-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.555-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.555.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.555.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.567-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.567-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.567.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.567.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.568-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.568-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.568.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.568.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.569-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.569-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.569.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.569.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.585-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.585-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.585.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.585.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.586-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.586-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.586.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.586.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.587-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.587-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.587.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.587.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.629-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.629-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.629.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.629.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.630-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.630-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.630.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.630.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.631-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.631-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.631.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.631.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.636-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.636-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.636.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.636.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.641-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.641-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.641.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.641.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.644-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.644-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.644.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.644.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.645-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.645-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.645.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.645.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.646-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.646-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.646.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.646.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.647-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.647-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.647.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.647.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.648-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.648-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.648.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.648.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.649-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.649-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.649.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.649.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.650-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.650-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.650.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.650.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.651.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.651.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.652-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.652-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.652.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.652.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.689-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.689-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.689.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.689.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.696-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.696-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.696.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.696.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.746-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.746-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.746.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.746.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.750-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.750-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.750.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.750.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.751-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.751-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.751.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.751.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.766-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.766-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.766.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.766.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.767-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.767-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.767.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.767.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.768-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.768-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.768.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.768.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.7681-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7681-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.7681.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7681.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.7682-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7682-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.7682.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7682.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.7701-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7701-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.7701.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7701.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.7712-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7712-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.7712.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7712.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.7713-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7713-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.7713.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.7713.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.1.781-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.781-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.1.781.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.1.781.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.7.1.dev1-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.7.1.dev1-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.7.1.dev1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.7.1.dev1.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.7.1.dev2-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.7.1.dev2-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.7.1.dev2.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.7.1.dev2.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.7.1.dev3-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.7.1.dev3-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.7.1.dev3.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.7.1.dev3.tar.gz -------------------------------------------------------------------------------- /dist/litellm-0.9.2.dev1-py3-none-any.whl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.9.2.dev1-py3-none-any.whl -------------------------------------------------------------------------------- /dist/litellm-0.9.2.dev1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/dist/litellm-0.9.2.dev1.tar.gz -------------------------------------------------------------------------------- /docs/my-website/.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | 4 | # Production 5 | /build 6 | 7 | # Generated files 8 | .docusaurus 9 | .cache-loader 10 | 11 | # Misc 12 | .DS_Store 13 | .env.local 14 | .env.development.local 15 | .env.test.local 16 | .env.production.local 17 | 18 | npm-debug.log* 19 | yarn-debug.log* 20 | yarn-error.log* 21 | -------------------------------------------------------------------------------- /docs/my-website/README.md: -------------------------------------------------------------------------------- 1 | # Website 2 | 3 | This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. 4 | 5 | ### Installation 6 | 7 | ``` 8 | $ yarn 9 | ``` 10 | 11 | ### Local Development 12 | 13 | ``` 14 | $ yarn start 15 | ``` 16 | 17 | This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. 18 | 19 | ### Build 20 | 21 | ``` 22 | $ yarn build 23 | ``` 24 | 25 | This command generates static content into the `build` directory and can be served using any static contents hosting service. 26 | 27 | ### Deployment 28 | 29 | Using SSH: 30 | 31 | ``` 32 | $ USE_SSH=true yarn deploy 33 | ``` 34 | 35 | Not using SSH: 36 | 37 | ``` 38 | $ GIT_USER= yarn deploy 39 | ``` 40 | 41 | If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch. 42 | -------------------------------------------------------------------------------- /docs/my-website/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [require.resolve('@docusaurus/core/lib/babel/preset')], 3 | }; 4 | -------------------------------------------------------------------------------- /docs/my-website/blog/2021-08-26-welcome/index.md: -------------------------------------------------------------------------------- 1 | # 🚅 litellm 2 | A light 100 line package to simplify calling OpenAI, Azure, Cohere, Anthropic APIs 3 | 4 | ###### litellm manages: 5 | * Calling all LLM APIs using the OpenAI format - `completion(model, messages)` 6 | * Consistent output for all LLM APIs, text responses will always be available at `['choices'][0]['message']['content']` 7 | * Consistent Exceptions for all LLM APIs, we map RateLimit, Context Window, and Authentication Error exceptions across all providers to their OpenAI equivalents. [see Code](https://github.com/BerriAI/litellm/blob/ba1079ff6698ef238c5c7f771dd2b698ec76f8d9/litellm/utils.py#L250) 8 | 9 | ###### observability: 10 | * Logging - see exactly what the raw model request/response is by plugging in your own function `completion(.., logger_fn=your_logging_fn)` and/or print statements from the package `litellm.set_verbose=True` 11 | * Callbacks - automatically send your data to Helicone, Sentry, Posthog, Slack - `litellm.success_callbacks`, `litellm.failure_callbacks` [see Callbacks](https://litellm.readthedocs.io/en/latest/advanced/) 12 | 13 | ## Quick Start 14 | Go directly to code: [Getting Started Notebook](https://colab.research.google.com/drive/1gR3pY-JzDZahzpVdbGBtrNGDBmzUNJaJ?usp=sharing) 15 | ### Installation 16 | ``` 17 | pip install litellm 18 | ``` 19 | 20 | ### Usage 21 | ```python 22 | from litellm import completion 23 | 24 | ## set ENV variables 25 | os.environ["OPENAI_API_KEY"] = "openai key" 26 | os.environ["COHERE_API_KEY"] = "cohere key" 27 | 28 | messages = [{ "content": "Hello, how are you?","role": "user"}] 29 | 30 | # openai call 31 | response = completion(model="gpt-3.5-turbo", messages=messages) 32 | 33 | # cohere call 34 | response = completion("command-nightly", messages) 35 | ``` 36 | Need Help / Support : [see troubleshooting](https://litellm.readthedocs.io/en/latest/troubleshoot) 37 | 38 | ## Why did we build liteLLM 39 | - **Need for simplicity**: Our code started to get extremely complicated managing & translating calls between Azure, OpenAI, Cohere 40 | 41 | ## Support 42 | * [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) 43 | * Contact us at ishaan@berri.ai / krrish@berri.ai 44 | -------------------------------------------------------------------------------- /docs/my-website/docs/caching/caching_api.md: -------------------------------------------------------------------------------- 1 | # Hosted Cache - api.litellm.ai 2 | 3 | Use api.litellm.ai for caching `completion()` and `embedding()` responses 4 | 5 | ## Quick Start Usage - Completion 6 | ```python 7 | import litellm 8 | from litellm import completion 9 | from litellm.caching import Cache 10 | litellm.cache = Cache(type="hosted") # init cache to use api.litellm.ai 11 | 12 | # Make completion calls 13 | response1 = completion( 14 | model="gpt-3.5-turbo", 15 | messages=[{"role": "user", "content": "Tell me a joke."}] 16 | caching=True 17 | ) 18 | 19 | response2 = completion( 20 | model="gpt-3.5-turbo", 21 | messages=[{"role": "user", "content": "Tell me a joke."}], 22 | caching=True 23 | ) 24 | # response1 == response2, response 1 is cached 25 | ``` 26 | 27 | 28 | ## Usage - Embedding() 29 | 30 | ```python 31 | import time 32 | import litellm 33 | from litellm import completion, embedding 34 | from litellm.caching import Cache 35 | litellm.cache = Cache(type="hosted") 36 | 37 | start_time = time.time() 38 | embedding1 = embedding(model="text-embedding-ada-002", input=["hello from litellm"*5], caching=True) 39 | end_time = time.time() 40 | print(f"Embedding 1 response time: {end_time - start_time} seconds") 41 | 42 | start_time = time.time() 43 | embedding2 = embedding(model="text-embedding-ada-002", input=["hello from litellm"*5], caching=True) 44 | end_time = time.time() 45 | print(f"Embedding 2 response time: {end_time - start_time} seconds") 46 | ``` 47 | 48 | ## Caching with Streaming 49 | LiteLLM can cache your streamed responses for you 50 | 51 | ### Usage 52 | ```python 53 | import litellm 54 | from litellm import completion 55 | from litellm.caching import Cache 56 | litellm.cache = Cache(type="hosted") 57 | 58 | # Make completion calls 59 | response1 = completion( 60 | model="gpt-3.5-turbo", 61 | messages=[{"role": "user", "content": "Tell me a joke."}], 62 | stream=True, 63 | caching=True) 64 | for chunk in response1: 65 | print(chunk) 66 | 67 | 68 | response2 = completion( 69 | model="gpt-3.5-turbo", 70 | messages=[{"role": "user", "content": "Tell me a joke."}], 71 | stream=True, 72 | caching=True) 73 | for chunk in response2: 74 | print(chunk) 75 | ``` -------------------------------------------------------------------------------- /docs/my-website/docs/completion/config.md: -------------------------------------------------------------------------------- 1 | # Model Config 2 | 3 | Model-specific changes can make our code complicated, making it harder to debug errors. Use model configs to simplify this. 4 | 5 | ### usage 6 | 7 | Handling prompt logic. Different models have different context windows. Use `adapt_to_prompt_size` to select the right model for the prompt (in case the current model is too small). 8 | 9 | 10 | ```python 11 | from litellm import completion_with_config 12 | import os 13 | 14 | config = { 15 | "available_models": ["gpt-3.5-turbo", "claude-instant-1", "gpt-3.5-turbo-16k"], 16 | "adapt_to_prompt_size": True, # 👈 key change 17 | } 18 | 19 | # set env var 20 | os.environ["OPENAI_API_KEY"] = "your-api-key" 21 | os.environ["ANTHROPIC_API_KEY"] = "your-api-key" 22 | 23 | 24 | sample_text = "how does a court case get to the Supreme Court?" * 1000 25 | messages = [{"content": sample_text, "role": "user"}] 26 | response = completion_with_config(model="gpt-3.5-turbo", messages=messages, config=config) 27 | ``` 28 | 29 | [**See Code**](https://github.com/BerriAI/litellm/blob/30724d9e51cdc2c3e0eb063271b4f171bc01b382/litellm/utils.py#L2783) 30 | 31 | ### Complete Config Structure 32 | 33 | ```python 34 | config = { 35 | "default_fallback_models": # [Optional] List of model names to try if a call fails 36 | "available_models": # [Optional] List of all possible models you could call 37 | "adapt_to_prompt_size": # [Optional] True/False - if you want to select model based on prompt size (will pick from available_models) 38 | "model": { 39 | "model-name": { 40 | "needs_moderation": # [Optional] True/False - if you want to call openai moderations endpoint before making completion call. Will raise exception, if flagged. 41 | "error_handling": { 42 | "error-type": { # One of the errors listed here - https://docs.litellm.ai/docs/exception_mapping#custom-mapping-list 43 | "fallback_model": "" # str, name of the model it should try instead, when that error occurs 44 | } 45 | } 46 | } 47 | } 48 | } 49 | ``` -------------------------------------------------------------------------------- /docs/my-website/docs/completion/message_trimming.md: -------------------------------------------------------------------------------- 1 | # Trimming Input Messages 2 | **Use litellm.trim_messages() to ensure messages does not exceed a model's token limit or specified `max_tokens`** 3 | 4 | ## Usage 5 | ```python 6 | from litellm import completion 7 | from litellm.utils import trim_messages 8 | 9 | response = completion( 10 | model=model, 11 | messages=trim_messages(messages, model) # trim_messages ensures tokens(messages) < max_tokens(model) 12 | ) 13 | ``` 14 | 15 | ## Usage - set max_tokens 16 | ```python 17 | from litellm import completion 18 | from litellm.utils import trim_messages 19 | 20 | response = completion( 21 | model=model, 22 | messages=trim_messages(messages, model, max_tokens=10), # trim_messages ensures tokens(messages) < max_tokens 23 | ) 24 | ``` 25 | 26 | ## Parameters 27 | 28 | The function uses the following parameters: 29 | 30 | - `messages`:[Required] This should be a list of input messages 31 | 32 | - `model`:[Optional] This is the LiteLLM model being used. This parameter is optional, as you can alternatively specify the `max_tokens` parameter. 33 | 34 | - `system_message`:[Optional] This is a string containing an optional system message that will be preserved at the beginning of the conversation. This parameter is optional and set to `None` by default. 35 | 36 | - `max_tokens`:[Optional] This is an int, manually set upper limit on messages 37 | 38 | - `trim_ratio`:[Optional] This represents the target ratio of tokens to use following trimming. It's default value is 0.75, which implies that messages will be trimmed to utilise about 75% -------------------------------------------------------------------------------- /docs/my-website/docs/completion/model_alias.md: -------------------------------------------------------------------------------- 1 | # Model Alias 2 | 3 | The model name you show an end-user might be different from the one you pass to LiteLLM - e.g. Displaying `GPT-3.5` while calling `gpt-3.5-turbo-16k` on the backend. 4 | 5 | LiteLLM simplifies this by letting you pass in a model alias mapping. 6 | 7 | # expected format 8 | 9 | ```python 10 | litellm.model_alias_map = { 11 | # a dictionary containing a mapping of the alias string to the actual litellm model name string 12 | "model_alias": "litellm_model_name" 13 | } 14 | ``` 15 | 16 | # usage 17 | 18 | ### Relevant Code 19 | ```python 20 | model_alias_map = { 21 | "GPT-3.5": "gpt-3.5-turbo-16k", 22 | "llama2": "replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf" 23 | } 24 | 25 | litellm.model_alias_map = model_alias_map 26 | ``` 27 | 28 | ### Complete Code 29 | ```python 30 | import litellm 31 | from litellm import completion 32 | 33 | 34 | ## set ENV variables 35 | os.environ["OPENAI_API_KEY"] = "openai key" 36 | os.environ["REPLICATE_API_KEY"] = "cohere key" 37 | 38 | ## set model alias map 39 | model_alias_map = { 40 | "GPT-3.5": "gpt-3.5-turbo-16k", 41 | "llama2": "replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf" 42 | } 43 | 44 | litellm.model_alias_map = model_alias_map 45 | 46 | messages = [{ "content": "Hello, how are you?","role": "user"}] 47 | 48 | # openai call 49 | response = completion(model="GPT-3.5", messages=messages) 50 | 51 | # replicate call 52 | response = completion("llama2", messages) 53 | ``` 54 | 55 | 56 | # no-code 57 | 58 | If you use litellm client, you can also do this without going into code. [Learn more]("https://docs.litellm.ai/docs/debugging/hosted_debugging") -------------------------------------------------------------------------------- /docs/my-website/docs/completion/multiple_deployments.md: -------------------------------------------------------------------------------- 1 | # Multiple Deployments 2 | 3 | If you have multiple deployments of the same model, you can pass the list of deployments, and LiteLLM will return the first result. 4 | 5 | ## Quick Start 6 | 7 | Multiple providers offer Mistral-7B-Instruct. 8 | 9 | Here's how you can use litellm to return the first result: 10 | 11 | ```python 12 | from litellm import completion 13 | 14 | messages=[{"role": "user", "content": "Hey, how's it going?"}] 15 | 16 | ## All your mistral deployments ## 17 | model_list = [{ 18 | "model_name": "mistral-7b-instruct", 19 | "litellm_params": { # params for litellm completion/embedding call 20 | "model": "replicate/mistralai/mistral-7b-instruct-v0.1:83b6a56e7c828e667f21fd596c338fd4f0039b46bcfa18d973e8e70e455fda70", 21 | "api_key": "replicate_api_key", 22 | } 23 | }, { 24 | "model_name": "mistral-7b-instruct", 25 | "litellm_params": { # params for litellm completion/embedding call 26 | "model": "together_ai/mistralai/Mistral-7B-Instruct-v0.1", 27 | "api_key": "togetherai_api_key", 28 | } 29 | }, { 30 | "model_name": "mistral-7b-instruct", 31 | "litellm_params": { # params for litellm completion/embedding call 32 | "model": "together_ai/mistralai/Mistral-7B-Instruct-v0.1", 33 | "api_key": "togetherai_api_key", 34 | } 35 | }, { 36 | "model_name": "mistral-7b-instruct", 37 | "litellm_params": { # params for litellm completion/embedding call 38 | "model": "perplexity/mistral-7b-instruct", 39 | "api_key": "perplexity_api_key" 40 | } 41 | }, { 42 | "model_name": "mistral-7b-instruct", 43 | "litellm_params": { 44 | "model": "deepinfra/mistralai/Mistral-7B-Instruct-v0.1", 45 | "api_key": "deepinfra_api_key" 46 | } 47 | }] 48 | 49 | ## LiteLLM completion call ## returns first response 50 | response = completion(model="mistral-7b-instruct", messages=messages, model_list=model_list) 51 | 52 | print(response) 53 | ``` -------------------------------------------------------------------------------- /docs/my-website/docs/completion/output.md: -------------------------------------------------------------------------------- 1 | # Output 2 | 3 | ## Format 4 | Here's the exact json output and type you can expect from all litellm `completion` calls for all models 5 | 6 | ```python 7 | { 8 | 'choices': [ 9 | { 10 | 'finish_reason': str, # String: 'stop' 11 | 'index': int, # Integer: 0 12 | 'message': { # Dictionary [str, str] 13 | 'role': str, # String: 'assistant' 14 | 'content': str # String: "default message" 15 | } 16 | } 17 | ], 18 | 'created': str, # String: None 19 | 'model': str, # String: None 20 | 'usage': { # Dictionary [str, int] 21 | 'prompt_tokens': int, # Integer 22 | 'completion_tokens': int, # Integer 23 | 'total_tokens': int # Integer 24 | } 25 | } 26 | 27 | ``` 28 | 29 | You can access the response as a dictionary or as a class object, just as OpenAI allows you 30 | ```python 31 | print(response.choices[0].message.content) 32 | print(response['choices'][0]['message']['content']) 33 | ``` 34 | 35 | Here's what an example response looks like 36 | ```python 37 | { 38 | 'choices': [ 39 | { 40 | 'finish_reason': 'stop', 41 | 'index': 0, 42 | 'message': { 43 | 'role': 'assistant', 44 | 'content': " I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic." 45 | } 46 | } 47 | ], 48 | 'created': 1691429984.3852863, 49 | 'model': 'claude-instant-1', 50 | 'usage': {'prompt_tokens': 18, 'completion_tokens': 23, 'total_tokens': 41} 51 | } 52 | ``` 53 | 54 | ## Additional Attributes 55 | 56 | You can also access information like latency. 57 | 58 | ```python 59 | from litellm import completion 60 | import os 61 | os.environ["ANTHROPIC_API_KEY"] = "your-api-key" 62 | 63 | messages=[{"role": "user", "content": "Hey!"}] 64 | 65 | response = completion(model="claude-2", messages=messages) 66 | 67 | print(response.response_ms) # 616.25# 616.25 68 | ``` -------------------------------------------------------------------------------- /docs/my-website/docs/contact.md: -------------------------------------------------------------------------------- 1 | # Contact Us 2 | 3 | [![](https://dcbadge.vercel.app/api/server/wuPM9dRgDw)](https://discord.gg/wuPM9dRgDw) 4 | 5 | * [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) 6 | * Contact us at ishaan@berri.ai / krrish@berri.ai 7 | -------------------------------------------------------------------------------- /docs/my-website/docs/default_code_snippet.md: -------------------------------------------------------------------------------- 1 | --- 2 | displayed_sidebar: tutorialSidebar 3 | --- 4 | # Get Started 5 | 6 | import QueryParamReader from '../src/components/queryParamReader.js' 7 | import TokenComponent from '../src/components/queryParamToken.js' 8 | 9 | :::info 10 | 11 | This section assumes you've already added your API keys in 12 | 13 | If you want to use the non-hosted version, [go here](https://docs.litellm.ai/docs/#quick-start) 14 | 15 | ::: 16 | 17 | 18 | ``` 19 | pip install litellm 20 | ``` 21 | 22 | -------------------------------------------------------------------------------- /docs/my-website/docs/embedding/moderation.md: -------------------------------------------------------------------------------- 1 | # Moderation 2 | LiteLLM supports the moderation endpoint for OpenAI 3 | 4 | ## Usage 5 | ```python 6 | import os 7 | from litellm import moderation 8 | os.environ['OPENAI_API_KEY'] = "" 9 | response = moderation(input="i'm ishaan cto of litellm") 10 | ``` 11 | -------------------------------------------------------------------------------- /docs/my-website/docs/extras/contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing to Documentation 2 | 3 | Clone litellm 4 | ``` 5 | git clone https://github.com/BerriAI/litellm.git 6 | ``` 7 | 8 | ### Local setup for locally running docs 9 | 10 | #### Installation 11 | ``` 12 | pip install mkdocs 13 | ``` 14 | 15 | #### Locally Serving Docs 16 | ``` 17 | mkdocs serve 18 | ``` 19 | If you see `command not found: mkdocs` try running the following 20 | ``` 21 | python3 -m mkdocs serve 22 | ``` 23 | 24 | This command builds your Markdown files into HTML and starts a development server to browse your documentation. Open up [http://127.0.0.1:8000/](http://127.0.0.1:8000/) in your web browser to see your documentation. You can make changes to your Markdown files and your docs will automatically rebuild. 25 | 26 | [Full tutorial here](https://docs.readthedocs.io/en/stable/intro/getting-started-with-mkdocs.html) 27 | 28 | ### Making changes to Docs 29 | - All the docs are placed under the `docs` directory 30 | - If you are adding a new `.md` file or editing the hierarchy edit `mkdocs.yml` in the root of the project 31 | - After testing your changes, make a change to the `main` branch of [github.com/BerriAI/litellm](https://github.com/BerriAI/litellm) 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /docs/my-website/docs/migration.md: -------------------------------------------------------------------------------- 1 | # Migration 2 | 3 | When we have breaking changes (i.e. going from 1.x.x to 2.x.x), we will document those changes here. 4 | 5 | 6 | 🧹 nothing here yet.. 7 | 8 | 9 | **How can we communicate changes better?** 10 | Tell us 11 | - [Discord](https://discord.com/invite/wuPM9dRgDw) 12 | - Email (krrish@berri.ai/ishaan@berri.ai) 13 | - Text us (+17708783106) 14 | -------------------------------------------------------------------------------- /docs/my-website/docs/observability/callbacks.md: -------------------------------------------------------------------------------- 1 | # Callbacks 2 | 3 | ## Use Callbacks to send Output Data to Posthog, Sentry etc 4 | 5 | liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses. 6 | 7 | liteLLM supports: 8 | 9 | - [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback) 10 | - [LLMonitor](https://llmonitor.com/docs) 11 | - [Helicone](https://docs.helicone.ai/introduction) 12 | - [Traceloop](https://traceloop.com/docs) 13 | - [Sentry](https://docs.sentry.io/platforms/python/) 14 | - [PostHog](https://posthog.com/docs/libraries/python) 15 | - [Slack](https://slack.dev/bolt-python/concepts) 16 | 17 | ### Quick Start 18 | 19 | ```python 20 | from litellm import completion 21 | 22 | # set callbacks 23 | litellm.input_callback=["sentry"] # for sentry breadcrumbing - logs the input being sent to the api 24 | litellm.success_callback=["posthog", "helicone", "llmonitor"] 25 | litellm.failure_callback=["sentry", "llmonitor"] 26 | 27 | ## set env variables 28 | os.environ['SENTRY_API_URL'], os.environ['SENTRY_API_TRACE_RATE']= "" 29 | os.environ['POSTHOG_API_KEY'], os.environ['POSTHOG_API_URL'] = "api-key", "api-url" 30 | os.environ["HELICONE_API_KEY"] = "" 31 | os.environ["TRACELOOP_API_KEY"] = "" 32 | os.environ["LLMONITOR_APP_ID"] = "" 33 | 34 | response = completion(model="gpt-3.5-turbo", messages=messages) 35 | ``` 36 | -------------------------------------------------------------------------------- /docs/my-website/docs/observability/helicone_integration.md: -------------------------------------------------------------------------------- 1 | # Helicone Tutorial 2 | [Helicone](https://helicone.ai/) is an open source observability platform that proxies your OpenAI traffic and provides you key insights into your spend, latency and usage. 3 | 4 | ## Use Helicone to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM) 5 | liteLLM provides `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses. 6 | 7 | In this case, we want to log requests to Helicone when a request succeeds. 8 | 9 | ### Approach 1: Use Callbacks 10 | Use just 1 line of code, to instantly log your responses **across all providers** with helicone: 11 | ```python 12 | litellm.success_callback=["helicone"] 13 | ``` 14 | 15 | Complete code 16 | ```python 17 | from litellm import completion 18 | 19 | ## set env variables 20 | os.environ["HELICONE_API_KEY"] = "your-helicone-key" 21 | os.environ["OPENAI_API_KEY"], os.environ["COHERE_API_KEY"] = "", "" 22 | 23 | # set callbacks 24 | litellm.success_callback=["helicone"] 25 | 26 | #openai call 27 | response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) 28 | 29 | #cohere call 30 | response = completion(model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}]) 31 | ``` 32 | 33 | ### Approach 2: [OpenAI + Azure only] Use Helicone as a proxy 34 | Helicone provides advanced functionality like caching, etc. Helicone currently supports this for Azure and OpenAI. 35 | 36 | If you want to use Helicone to proxy your OpenAI/Azure requests, then you can - 37 | 38 | - Set helicone as your base url via: `litellm.api_url` 39 | - Pass in helicone request headers via: `litellm.headers` 40 | 41 | Complete Code 42 | ```python 43 | import litellm 44 | from litellm import completion 45 | 46 | litellm.api_base = "https://oai.hconeai.com/v1" 47 | litellm.headers = {"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}"} 48 | 49 | response = litellm.completion( 50 | model="gpt-3.5-turbo", 51 | messages=[{"role": "user", "content": "how does a court case get to the Supreme Court?"}] 52 | ) 53 | 54 | print(response) 55 | ``` 56 | -------------------------------------------------------------------------------- /docs/my-website/docs/observability/sentry.md: -------------------------------------------------------------------------------- 1 | # Sentry Tutorial 2 | [Sentry](https://sentry.io/) provides error monitoring for production. LiteLLM can add breadcrumbs and send exceptions to Sentry with this integration 3 | 4 | This works on normal, async and streaming completion calls 5 | 6 | ### usage 7 | 8 | ```python 9 | import litellm 10 | from litellm import completion 11 | litellm.set_verbose = True 12 | 13 | litellm.input_callback=["sentry"] # adds sentry breadcrumbing 14 | litellm.failure_callback=["sentry"] # [OPTIONAL] if you want litellm to capture -> send exception to sentry 15 | 16 | import os 17 | os.environ["SENTRY_API_URL"] = "your-sentry-url" 18 | os.environ["OPENAI_API_KEY"] = "your-openai-key" 19 | 20 | # set bad key to trigger error 21 | api_key="bad-key" 22 | response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey!"}], stream=True, api_key=api_key) 23 | 24 | print(response) 25 | ``` 26 | 27 | [Let us know](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+) if you need any additional options from Sentry. 28 | 29 | -------------------------------------------------------------------------------- /docs/my-website/docs/observability/telemetry.md: -------------------------------------------------------------------------------- 1 | # Telemetry 2 | 3 | LiteLLM contains a telemetry feature that tells us what models are used, and what errors are hit. 4 | 5 | ## What is logged? 6 | 7 | Only the model name and exception raised is logged. 8 | 9 | ## Why? 10 | We use this information to help us understand how LiteLLM is used, and improve stability. 11 | 12 | ## Opting out 13 | If you prefer to opt out of telemetry, you can do this by setting `litellm.telemetry = False`. -------------------------------------------------------------------------------- /docs/my-website/docs/observability/traceloop_integration.md: -------------------------------------------------------------------------------- 1 | import Image from '@theme/IdealImage'; 2 | 3 | # Traceloop (OpenLLMetry) - Tracing LLMs with OpenTelemetry 4 | 5 | [Traceloop](https://traceloop.com) is a platform for monitoring and debugging the quality of your LLM outputs. 6 | It provides you with a way to track the performance of your LLM application; rollout changes with confidence; and debug issues in production. 7 | It is based on [OpenTelemetry](https://opentelemetry.io), so it can provide full visibility to your LLM requests, as well vector DB usage, and other infra in your stack. 8 | 9 | 10 | 11 | ## Getting Started 12 | 13 | First, sign up to get an API key on the [Traceloop dashboard](https://app.traceloop.com/settings/api-keys). 14 | 15 | Then, install the Traceloop SDK: 16 | 17 | ``` 18 | pip install traceloop-sdk 19 | ``` 20 | 21 | Use just 1 line of code, to instantly log your LLM responses: 22 | 23 | ```python 24 | litellm.success_callback = ["traceloop"] 25 | ``` 26 | 27 | When running your app, make sure to set the `TRACELOOP_API_KEY` environment variable to your API key. 28 | 29 | To get better visualizations on how your code behaves, you may want to annotate specific parts of your LLM chain. See [Traceloop docs on decorators](https://traceloop.com/docs/python-sdk/decorators) for more information. 30 | 31 | ## Exporting traces to other systems (e.g. Datadog, New Relic, and others) 32 | 33 | Since Traceloop SDK uses OpenTelemetry to send data, you can easily export your traces to other systems, such as Datadog, New Relic, and others. See [Traceloop docs on exporters](https://traceloop.com/docs/python-sdk/exporters) for more information. 34 | 35 | ## Support 36 | 37 | For any question or issue with integration you can reach out to the Traceloop team on [Slack](https://join.slack.com/t/traceloopcommunity/shared_invite/zt-1plpfpm6r-zOHKI028VkpcWdobX65C~g) or via [email](mailto:dev@traceloop.com). 38 | -------------------------------------------------------------------------------- /docs/my-website/docs/observability/wandb_integration.md: -------------------------------------------------------------------------------- 1 | import Image from '@theme/IdealImage'; 2 | 3 | # Weights & Biases - Logging LLM Input/Output 4 | Weights & Biases helps AI developers build better models faster https://wandb.ai 5 | 6 | 7 | 8 | :::info 9 | We want to learn how we can make the callbacks better! Meet the LiteLLM [founders](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) or 10 | join our [discord](https://discord.gg/wuPM9dRgDw) 11 | ::: 12 | 13 | ## Pre-Requisites 14 | Ensure you have run `pip install wandb` for this integration 15 | ```shell 16 | pip install wandb litellm 17 | ``` 18 | 19 | ## Quick Start 20 | Use just 2 lines of code, to instantly log your responses **across all providers** with Weights & Biases 21 | 22 | ```python 23 | litellm.success_callback = ["wandb"] 24 | ``` 25 | ```python 26 | # pip install wandb 27 | import litellm 28 | import os 29 | 30 | os.environ["WANDB_API_KEY"] = "" 31 | # LLM API Keys 32 | os.environ['OPENAI_API_KEY']="" 33 | 34 | # set wandb as a callback, litellm will send the data to Weights & Biases 35 | litellm.success_callback = ["wandb"] 36 | 37 | # openai call 38 | response = litellm.completion( 39 | model="gpt-3.5-turbo", 40 | messages=[ 41 | {"role": "user", "content": "Hi 👋 - i'm openai"} 42 | ] 43 | ) 44 | ``` 45 | 46 | ## Support & Talk to Founders 47 | 48 | - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) 49 | - [Community Discord 💭](https://discord.gg/wuPM9dRgDw) 50 | - Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ 51 | - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai -------------------------------------------------------------------------------- /docs/my-website/docs/projects.md: -------------------------------------------------------------------------------- 1 | # Projects Built on LiteLLM 2 | 3 | 4 | 5 | ### EntoAI 6 | Chat and Ask on your own data. 7 | [Github](https://github.com/akshata29/entaoai) 8 | 9 | ### GPT-Migrate 10 | Easily migrate your codebase from one framework or language to another. 11 | [Github](https://github.com/0xpayne/gpt-migrate) 12 | 13 | ### Otter 14 | Otter, a multi-modal model based on OpenFlamingo (open-sourced version of DeepMind's Flamingo), trained on MIMIC-IT and showcasing improved instruction-following and in-context learning ability. 15 | [Github](https://github.com/Luodian/Otter) 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /docs/my-website/docs/projects/Codium PR Agent.md: -------------------------------------------------------------------------------- 1 | An AI-Powered 🤖 Tool for Automated Pull Request Analysis, 2 | Feedback, Suggestions 💻🔍 3 | [Github](https://github.com/Codium-ai/pr-agent) -------------------------------------------------------------------------------- /docs/my-website/docs/projects/FastREPL.md: -------------------------------------------------------------------------------- 1 | ⚡Fast Run-Eval-Polish Loop for LLM Applications 2 | 3 | Core: https://github.com/fastrepl/fastrepl 4 | Proxy: https://github.com/fastrepl/proxy 5 | -------------------------------------------------------------------------------- /docs/my-website/docs/projects/GPT Migrate.md: -------------------------------------------------------------------------------- 1 | Easily migrate your codebase from one framework or language to another. -------------------------------------------------------------------------------- /docs/my-website/docs/projects/Langstream.md: -------------------------------------------------------------------------------- 1 | Build robust LLM applications with true composability 🔗 2 | [Github](https://github.com/rogeriochaves/langstream) 3 | [Docs](https://rogeriochaves.github.io/langstream/) -------------------------------------------------------------------------------- /docs/my-website/docs/projects/LiteLLM Proxy.md: -------------------------------------------------------------------------------- 1 | ### LiteLLM Proxy 2 | liteLLM Proxy Server: 50+ LLM Models, Error Handling, Caching 3 | [Github](https://github.com/BerriAI/litellm/tree/main/proxy-server) -------------------------------------------------------------------------------- /docs/my-website/docs/projects/OpenInterpreter.md: -------------------------------------------------------------------------------- 1 | Open Interpreter lets LLMs run code on your computer to complete tasks. 2 | [Github](https://github.com/KillianLucas/open-interpreter/) -------------------------------------------------------------------------------- /docs/my-website/docs/projects/Otter.md: -------------------------------------------------------------------------------- 1 | 🦦 Otter, a multi-modal model based on OpenFlamingo (open-sourced version of DeepMind's Flamingo), trained on MIMIC-IT and showcasing improved instruction-following and in-context learning ability. 2 | [Github](https://github.com/Luodian/Otter) -------------------------------------------------------------------------------- /docs/my-website/docs/projects/PROMPTMETHEUS.md: -------------------------------------------------------------------------------- 1 | 🔥 PROMPTMETHEUS – Prompt Engineering IDE 2 | 3 | Compose, test, optimize, and deploy reliable prompts for large language models. 4 | 5 | PROMPTMETHEUS is a Prompt Engineering IDE, designed to help you automate repetitive tasks and augment your apps and workflows with the mighty capabilities of all the LLMs in the LiteLLM quiver. 6 | 7 | Website → [www.promptmetheus.com](https://promptmetheus.com) 8 | FORGE → [forge.promptmetheus.com](https://forge.promptmetheus.com) 9 | ARCHERY → [archery.promptmetheus.com](https://archery.promptmetheus.com) 10 | -------------------------------------------------------------------------------- /docs/my-website/docs/projects/Prompt2Model.md: -------------------------------------------------------------------------------- 1 | Prompt2Model - Generate Deployable Models from Instructions 2 | 3 | Github: https://github.com/neulab/prompt2model 4 | 5 | Prompt2Model is a system that takes a natural language task description (like the prompts used for LLMs such as ChatGPT) to train a small special-purpose model that is conducive for deployment. -------------------------------------------------------------------------------- /docs/my-website/docs/projects/Quivr.md: -------------------------------------------------------------------------------- 1 | 🧠 Your Second Brain supercharged by Generative AI 🧠 Dump all your files and chat with your personal assistant on your files & more using GPT 3.5/4, Private, Anthropic, VertexAI, LLMs... -------------------------------------------------------------------------------- /docs/my-website/docs/projects/SalesGPT.md: -------------------------------------------------------------------------------- 1 | 🤖 SalesGPT - Your Context-Aware AI Sales Assistant 2 | 3 | Github: https://github.com/filip-michalsky/SalesGPT -------------------------------------------------------------------------------- /docs/my-website/docs/projects/YiVal.md: -------------------------------------------------------------------------------- 1 | 🚀 Evaluate and Evolve.🚀 YiVal is an open source GenAI-Ops framework that allows you to manually or automatically tune and evaluate your AIGC prompts, retrieval configs and fine-tune the model params all at once with your preferred choices of test dataset generation, evaluation algorithms and improvement strategies. 2 | 3 | Github: https://github.com/YiVal/YiVal 4 | 5 | Docs: https://yival.github.io/YiVal/ -------------------------------------------------------------------------------- /docs/my-website/docs/providers/ai21.md: -------------------------------------------------------------------------------- 1 | # AI21 2 | 3 | LiteLLM supports j2-light, j2-mid and j2-ultra from [AI21](https://www.ai21.com/studio/pricing). 4 | 5 | They're available to use without a waitlist. 6 | 7 | ### API KEYS 8 | ```python 9 | import os 10 | os.environ["AI21_API_KEY"] = "your-api-key" 11 | ``` 12 | 13 | ### Sample Usage 14 | 15 | ```python 16 | from litellm import completion 17 | 18 | # set env variable 19 | os.environ["AI21_API_KEY"] = "your-api-key" 20 | 21 | messages = [{"role": "user", "content": "Write me a poem about the blue sky"}] 22 | 23 | completion(model="j2-light", messages=messages) 24 | ``` 25 | 26 | ### AI21 Models 27 | 28 | | Model Name | Function Call | Required OS Variables | 29 | |------------------|--------------------------------------------|--------------------------------------| 30 | | j2-light | `completion('j2-light', messages)` | `os.environ['AI21_API_KEY']` | 31 | | j2-mid | `completion('j2-mid', messages)` | `os.environ['AI21_API_KEY']` | 32 | | j2-ultra | `completion('j2-ultra', messages)` | `os.environ['AI21_API_KEY']` | -------------------------------------------------------------------------------- /docs/my-website/docs/providers/aleph_alpha.md: -------------------------------------------------------------------------------- 1 | # Aleph Alpha 2 | 3 | LiteLLM supports all models from [Aleph Alpha](https://www.aleph-alpha.com/). 4 | 5 | Like AI21 and Cohere, you can use these models without a waitlist. 6 | 7 | ### API KEYS 8 | ```python 9 | import os 10 | os.environ["ALEPHALPHA_API_KEY"] = "" 11 | ``` 12 | 13 | ### Aleph Alpha Models 14 | https://www.aleph-alpha.com/ 15 | 16 | | Model Name | Function Call | Required OS Variables | 17 | |------------------|--------------------------------------------|------------------------------------| 18 | | luminous-base | `completion(model='luminous-base', messages=messages)` | `os.environ['ALEPHALPHA_API_KEY']` | 19 | | luminous-base-control | `completion(model='luminous-base-control', messages=messages)` | `os.environ['ALEPHALPHA_API_KEY']` | 20 | | luminous-extended | `completion(model='luminous-extended', messages=messages)` | `os.environ['ALEPHALPHA_API_KEY']` | 21 | | luminous-extended-control | `completion(model='luminous-extended-control', messages=messages)` | `os.environ['ALEPHALPHA_API_KEY']` | 22 | | luminous-supreme | `completion(model='luminous-supreme', messages=messages)` | `os.environ['ALEPHALPHA_API_KEY']` | 23 | | luminous-supreme-control | `completion(model='luminous-supreme-control', messages=messages)` | `os.environ['ALEPHALPHA_API_KEY']` | 24 | -------------------------------------------------------------------------------- /docs/my-website/docs/providers/anthropic.md: -------------------------------------------------------------------------------- 1 | # Anthropic 2 | LiteLLM supports Claude-1, 1.2 and Claude-2. 3 | 4 | ## API Keys 5 | 6 | ```python 7 | import os 8 | 9 | os.environ["ANTHROPIC_API_KEY"] = "your-api-key" 10 | ``` 11 | 12 | ## Sample Usage 13 | 14 | ```python 15 | import os 16 | from litellm import completion 17 | 18 | # set env - [OPTIONAL] replace with your anthropic key 19 | os.environ["ANTHROPIC_API_KEY"] = "your-api-key" 20 | 21 | messages = [{"role": "user", "content": "Hey! how's it going?"}] 22 | response = completion(model="claude-instant-1", messages=messages) 23 | print(response) 24 | ``` 25 | 26 | ## streaming 27 | Just set `stream=True` when calling completion. 28 | 29 | ```python 30 | import os 31 | from litellm import completion 32 | 33 | # set env 34 | os.environ["ANTHROPIC_API_KEY"] = "your-api-key" 35 | 36 | messages = [{"role": "user", "content": "Hey! how's it going?"}] 37 | response = completion(model="claude-instant-1", messages=messages, stream=True) 38 | for chunk in response: 39 | print(chunk["choices"][0]["delta"]["content"]) # same as openai format 40 | ``` 41 | 42 | 43 | ### Model Details 44 | 45 | | Model Name | Function Call | Required OS Variables | 46 | |------------------|--------------------------------------------|--------------------------------------| 47 | | claude-instant-1 | `completion('claude-instant-1', messages)` | `os.environ['ANTHROPIC_API_KEY']` | 48 | | claude-instant-1.2 | `completion('claude-instant-1.2', messages)` | `os.environ['ANTHROPIC_API_KEY']` | 49 | | claude-2 | `completion('claude-2', messages)` | `os.environ['ANTHROPIC_API_KEY']` | 50 | -------------------------------------------------------------------------------- /docs/my-website/docs/providers/baseten.md: -------------------------------------------------------------------------------- 1 | # Baseten 2 | LiteLLM supports any Text-Gen-Interface models on Baseten. 3 | 4 | [Here's a tutorial on deploying a huggingface TGI model (Llama2, CodeLlama, WizardCoder, Falcon, etc.) on Baseten](https://truss.baseten.co/examples/performance/tgi-server) 5 | 6 | ### API KEYS 7 | ```python 8 | import os 9 | os.environ["BASETEN_API_KEY"] = "" 10 | ``` 11 | 12 | ### Baseten Models 13 | Baseten provides infrastructure to deploy and serve ML models https://www.baseten.co/. Use liteLLM to easily call models deployed on Baseten. 14 | 15 | Example Baseten Usage - Note: liteLLM supports all models deployed on Baseten 16 | 17 | Usage: Pass `model=baseten/` 18 | 19 | | Model Name | Function Call | Required OS Variables | 20 | |------------------|--------------------------------------------|------------------------------------| 21 | | Falcon 7B | `completion(model='baseten/qvv0xeq', messages=messages)` | `os.environ['BASETEN_API_KEY']` | 22 | | Wizard LM | `completion(model='baseten/q841o8w', messages=messages)` | `os.environ['BASETEN_API_KEY']` | 23 | | MPT 7B Base | `completion(model='baseten/31dxrj3', messages=messages)` | `os.environ['BASETEN_API_KEY']` | 24 | -------------------------------------------------------------------------------- /docs/my-website/docs/providers/cohere.md: -------------------------------------------------------------------------------- 1 | # Cohere 2 | 3 | LiteLLM supports 'command', 'command-light', 'command-medium', 'command-medium-beta', 'command-xlarge-beta', 'command-nightly' models from [Cohere](https://cohere.com/). 4 | 5 | Like AI21, these models are available without a waitlist. 6 | 7 | ### API KEYS 8 | 9 | ```python 10 | import os 11 | os.environ["COHERE_API_KEY"] = "" 12 | ``` 13 | 14 | ### Example Usage 15 | 16 | ```python 17 | 18 | from litellm import completion 19 | 20 | ## set ENV variables 21 | os.environ["COHERE_API_KEY"] = "cohere key" 22 | 23 | messages = [{ "content": "Hello, how are you?","role": "user"}] 24 | 25 | # cohere call 26 | response = completion("command-nightly", messages) 27 | ``` -------------------------------------------------------------------------------- /docs/my-website/docs/providers/custom_openai_proxy.md: -------------------------------------------------------------------------------- 1 | # Custom API Server (OpenAI Format) 2 | 3 | LiteLLM allows you to call your custom endpoint in the OpenAI ChatCompletion format 4 | 5 | ### API KEYS 6 | No api keys required 7 | 8 | ### Example Usage 9 | 10 | #### Pre-Requisites 11 | Ensure your proxy server has the following route 12 | Here's an example OpenAI proxy server with routes: https://replit.com/@BerriAI/openai-proxy#main.py 13 | 14 | ```python 15 | @app.route('/chat/completions', methods=["POST"]) 16 | def chat_completion(): 17 | print("got request for chat completion") 18 | 19 | ``` 20 | 21 | In order to use your custom OpenAI Chat Completion proxy with LiteLLM, ensure you set 22 | 23 | * `api_base` to your proxy url, example "https://openai-proxy.berriai.repl.co" 24 | * `custom_llm_provider` to `openai` this ensures litellm uses the `openai.ChatCompletion` to your api_base 25 | 26 | ```python 27 | import os 28 | from litellm import completion 29 | 30 | ## set ENV variables 31 | os.environ["OPENAI_API_KEY"] = "set anything here - key is not used for proxy" 32 | 33 | messages = [{ "content": "Hello, how are you?","role": "user"}] 34 | 35 | response = completion( 36 | model="command-nightly", 37 | messages=[{ "content": "Hello, how are you?","role": "user"}], 38 | api_base="https://openai-proxy.berriai.repl.co", 39 | custom_llm_provider="openai", 40 | temperature=0.2, 41 | max_tokens=80, 42 | ) 43 | print(response) 44 | ``` -------------------------------------------------------------------------------- /docs/my-website/docs/providers/deepinfra.md: -------------------------------------------------------------------------------- 1 | # DeepInfra 2 | https://deepinfra.com/ 3 | 4 | ## API Key 5 | ```python 6 | # env variable 7 | os.environ['DEEPINFRA_API_KEY'] 8 | ``` 9 | 10 | ## Sample Usage 11 | ```python 12 | from litellm import completion 13 | import os 14 | 15 | os.environ['DEEPINFRA_API_KEY'] = "" 16 | response = completion( 17 | model="deepinfra/meta-llama/Llama-2-70b-chat-hf", 18 | messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}] 19 | ) 20 | ``` 21 | 22 | ## Sample Usage - Streaming 23 | ```python 24 | from litellm import completion 25 | import os 26 | 27 | os.environ['DEEPINFRA_API_KEY'] = "" 28 | response = completion( 29 | model="deepinfra/meta-llama/Llama-2-70b-chat-hf", 30 | messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}], 31 | stream=True 32 | ) 33 | 34 | for chunk in response: 35 | print(chunk) 36 | ``` 37 | 38 | ## Chat Models 39 | | Model Name | Function Call | 40 | |------------------|--------------------------------------| 41 | | meta-llama/Llama-2-70b-chat-hf | `completion(model="deepinfra/meta-llama/Llama-2-70b-chat-hf", messages)` | 42 | | meta-llama/Llama-2-7b-chat-hf | `completion(model="deepinfra/meta-llama/Llama-2-7b-chat-hf", messages)` | 43 | | meta-llama/Llama-2-13b-chat-hf | `completion(model="deepinfra/meta-llama/Llama-2-13b-chat-hf", messages)` | 44 | | codellama/CodeLlama-34b-Instruct-hf | `completion(model="deepinfra/codellama/CodeLlama-34b-Instruct-hf", messages)` | 45 | | mistralai/Mistral-7B-Instruct-v0.1 | `completion(model="deepinfra/mistralai/Mistral-7B-Instruct-v0.1", messages)` | 46 | | jondurbin/airoboros-l2-70b-gpt4-1.4.1 | `completion(model="deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1", messages)` | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /docs/my-website/docs/providers/nlp_cloud.md: -------------------------------------------------------------------------------- 1 | # NLP Cloud 2 | 3 | LiteLLM supports all LLMs on NLP Cloud. 4 | 5 | ## API Keys 6 | 7 | ```python 8 | import os 9 | 10 | os.environ["NLP_CLOUD_API_KEY"] = "your-api-key" 11 | ``` 12 | 13 | ## Sample Usage 14 | 15 | ```python 16 | import os 17 | from litellm import completion 18 | 19 | # set env 20 | os.environ["NLP_CLOUD_API_KEY"] = "your-api-key" 21 | 22 | messages = [{"role": "user", "content": "Hey! how's it going?"}] 23 | response = completion(model="dolphin", messages=messages) 24 | print(response) 25 | ``` 26 | 27 | ## streaming 28 | Just set `stream=True` when calling completion. 29 | 30 | ```python 31 | import os 32 | from litellm import completion 33 | 34 | # set env 35 | os.environ["NLP_CLOUD_API_KEY"] = "your-api-key" 36 | 37 | messages = [{"role": "user", "content": "Hey! how's it going?"}] 38 | response = completion(model="dolphin", messages=messages, stream=True) 39 | for chunk in response: 40 | print(chunk["choices"][0]["delta"]["content"]) # same as openai format 41 | ``` 42 | 43 | ## non-dolphin models 44 | 45 | By default, LiteLLM will map `dolphin` and `chatdolphin` to nlp cloud. 46 | 47 | If you're trying to call any other model (e.g. GPT-J, Llama-2, etc.) with nlp cloud, just set it as your custom llm provider. 48 | 49 | 50 | ```python 51 | import os 52 | from litellm import completion 53 | 54 | # set env - [OPTIONAL] replace with your nlp cloud key 55 | os.environ["NLP_CLOUD_API_KEY"] = "your-api-key" 56 | 57 | messages = [{"role": "user", "content": "Hey! how's it going?"}] 58 | 59 | # e.g. to call Llama2 on NLP Cloud 60 | response = completion(model="nlp_cloud/finetuned-llama-2-70b", messages=messages, stream=True) 61 | for chunk in response: 62 | print(chunk["choices"][0]["delta"]["content"]) # same as openai format 63 | ``` 64 | -------------------------------------------------------------------------------- /docs/my-website/docs/providers/palm.md: -------------------------------------------------------------------------------- 1 | # PaLM API - Google 2 | 3 | ## Pre-requisites 4 | * `pip install -q google-generativeai` 5 | 6 | ## Sample Usage 7 | ```python 8 | import litellm 9 | import os 10 | 11 | os.environ['PALM_API_KEY'] = "" 12 | response = completion( 13 | model="palm/chat-bison", 14 | messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}] 15 | ) 16 | ``` 17 | 18 | ## Sample Usage - Streaming 19 | ```python 20 | import litellm 21 | import os 22 | 23 | os.environ['PALM_API_KEY'] = "" 24 | response = completion( 25 | model="palm/chat-bison", 26 | messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}], 27 | stream=True 28 | ) 29 | 30 | for chunk in response: 31 | print(chunk) 32 | ``` 33 | 34 | ## Chat Models 35 | | Model Name | Function Call | Required OS Variables | 36 | |------------------|--------------------------------------|-------------------------| 37 | | chat-bison | `completion('palm/chat-bison', messages)` | `os.environ['PALM_API_KEY']` | 38 | -------------------------------------------------------------------------------- /docs/my-website/docs/providers/perplexity.md: -------------------------------------------------------------------------------- 1 | # Perplexity AI (pplx-api) 2 | https://www.perplexity.ai 3 | 4 | ## API Key 5 | ```python 6 | # env variable 7 | os.environ['PERPLEXITYAI_API_KEY'] 8 | ``` 9 | 10 | ## Sample Usage 11 | ```python 12 | from litellm import completion 13 | import os 14 | 15 | os.environ['PERPLEXITYAI_API_KEY'] = "" 16 | response = completion( 17 | model="perplexity/mistral-7b-instruct", 18 | messages=messages 19 | ) 20 | print(response) 21 | ``` 22 | 23 | ## Sample Usage - Streaming 24 | ```python 25 | from litellm import completion 26 | import os 27 | 28 | os.environ['PERPLEXITYAI_API_KEY'] = "" 29 | response = completion( 30 | model="perplexity/mistral-7b-instruct", 31 | messages=messages, 32 | stream=True 33 | ) 34 | 35 | for chunk in response: 36 | print(chunk) 37 | ``` 38 | 39 | 40 | ## Supported Models 41 | All models listed here https://docs.perplexity.ai/docs/model-cards are supported 42 | 43 | | Model Name | Function Call | 44 | |--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| 45 | | codellama-34b-instruct | `completion(model="perplexity/codellama-34b-instruct", messages)` | 46 | | llama-2-13b-chat | `completion(model="perplexity/llama-2-13b-chat", messages)` | 47 | | llama-2-70b-chat | `completion(model="perplexity/llama-2-70b-chat", messages)` | 48 | | mistral-7b-instruct | `completion(model="perplexity/mistral-7b-instruct", messages)` | 49 | | replit-code-v1.5-3b | `completion(model="perplexity/replit-code-v1.5-3b", messages)` | 50 | 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /docs/my-website/docs/providers/petals.md: -------------------------------------------------------------------------------- 1 | # Petals 2 | Petals: https://github.com/bigscience-workshop/petals 3 | 4 | 5 | Open In Colab 6 | 7 | 8 | ## Pre-Requisites 9 | Ensure you have `petals` installed 10 | ```shell 11 | pip install git+https://github.com/bigscience-workshop/petals 12 | ``` 13 | 14 | ## Usage 15 | Ensure you add `petals/` as a prefix for all petals LLMs. This sets the custom_llm_provider to petals 16 | 17 | ```python 18 | from litellm import completion 19 | 20 | response = completion( 21 | model="petals/petals-team/StableBeluga2", 22 | messages=[{ "content": "Hello, how are you?","role": "user"}] 23 | ) 24 | 25 | print(response) 26 | ``` 27 | 28 | ## Usage with Streaming 29 | 30 | ```python 31 | response = completion( 32 | model="petals/petals-team/StableBeluga2", 33 | messages=[{ "content": "Hello, how are you?","role": "user"}], 34 | stream=True 35 | ) 36 | 37 | print(response) 38 | for chunk in response: 39 | print(chunk) 40 | ``` 41 | 42 | ### Model Details 43 | 44 | | Model Name | Function Call | 45 | |------------------|--------------------------------------------| 46 | | petals-team/StableBeluga | `completion('petals/petals-team/StableBeluga2', messages)` | 47 | | huggyllama/llama-65b | `completion('petals/huggyllama/llama-65b', messages)` | 48 | 49 | 50 | -------------------------------------------------------------------------------- /docs/my-website/docs/providers/replicate.md: -------------------------------------------------------------------------------- 1 | # Replicate 2 | 3 | LiteLLM supports all models on Replicate 4 | 5 | ### API KEYS 6 | ```python 7 | import os 8 | os.environ["REPLICATE_API_KEY"] = "" 9 | ``` 10 | 11 | 12 | ### Example Call 13 | 14 | ```python 15 | # !pip install litellm 16 | from litellm import completion 17 | import os 18 | ## set ENV variables 19 | os.environ["REPLICATE_API_KEY"] = "replicate key" 20 | 21 | messages = [{ "content": "Hello, how are you?","role": "user"}] 22 | 23 | # replicate llama-2 call 24 | response = completion( 25 | model="replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf", 26 | messages=messages 27 | ) 28 | ``` 29 | 30 | ### Replicate Models 31 | liteLLM supports all replicate LLMs 32 | 33 | For replicate models ensure to add a `replicate/` prefix to the `model` arg. liteLLM detects it using this arg. 34 | 35 | Below are examples on how to call replicate LLMs using liteLLM 36 | 37 | Model Name | Function Call | Required OS Variables | 38 | -----------------------------|----------------------------------------------------------------|--------------------------------------| 39 | replicate/llama-2-70b-chat | `completion('replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf', messages)` | `os.environ['REPLICATE_API_KEY']` | 40 | a16z-infra/llama-2-13b-chat| `completion('replicate/a16z-infra/llama-2-13b-chat:2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52', messages)`| `os.environ['REPLICATE_API_KEY']` | 41 | replicate/vicuna-13b | `completion('replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b', messages)` | `os.environ['REPLICATE_API_KEY']` | 42 | daanelson/flan-t5-large | `completion('replicate/daanelson/flan-t5-large:ce962b3f6792a57074a601d3979db5839697add2e4e02696b3ced4c022d4767f', messages)` | `os.environ['REPLICATE_API_KEY']` | 43 | custom-llm | Ensure the `model` param has `replicate/` as a prefix <`completion('replicate/custom-llm-version-id', messages)` | `os.environ['REPLICATE_API_KEY']` | 44 | 45 | -------------------------------------------------------------------------------- /docs/my-website/docs/troubleshoot.md: -------------------------------------------------------------------------------- 1 | # Support & Talk with founders 2 | [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) 3 | 4 | [Community Discord 💭](https://discord.gg/wuPM9dRgDw) 5 | 6 | Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ 7 | 8 | Our emails ✉️ ishaan@berri.ai / krrish@berri.ai 9 | 10 | [![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw) 11 | 12 | ## Stable Version 13 | 14 | If you're running into problems with installation / Usage 15 | Use the stable version of litellm 16 | 17 | ```shell 18 | pip install litellm==0.1.819 19 | ``` 20 | 21 | -------------------------------------------------------------------------------- /docs/my-website/docs/tutorials/finetuned_chat_gpt.md: -------------------------------------------------------------------------------- 1 | # Using Fine-Tuned gpt-3.5-turbo 2 | LiteLLM allows you to call `completion` with your fine-tuned gpt-3.5-turbo models 3 | If you're trying to create your custom finetuned gpt-3.5-turbo model following along on this tutorial: https://platform.openai.com/docs/guides/fine-tuning/preparing-your-dataset 4 | 5 | Once you've created your fine tuned model, you can call it with `completion()` 6 | 7 | ## Usage 8 | ```python 9 | import os 10 | from litellm import completion 11 | # set your OPENAI key in your .env as "OPENAI_API_KEY" 12 | 13 | response = completion( 14 | model="ft:gpt-3.5-turbo:my-org:custom_suffix:id", 15 | messages=[ 16 | {"role": "system", "content": "You are a helpful assistant."}, 17 | {"role": "user", "content": "Hello!"} 18 | ] 19 | ) 20 | 21 | print(response.choices[0].message) 22 | ``` -------------------------------------------------------------------------------- /docs/my-website/docs/tutorials/huggingface_codellama.md: -------------------------------------------------------------------------------- 1 | # CodeLlama - Code Infilling 2 | 3 | This tutorial shows how you can call CodeLlama (hosted on Huggingface PRO Inference Endpoints), to fill code. 4 | 5 | This is a specialized task particular to code models. The model is trained to generate the code (including comments) that best matches an existing prefix and suffix. 6 | 7 | This task is available in the base and instruction variants of the **7B** and **13B** CodeLlama models. It is not available for any of the 34B models or the Python versions. 8 | 9 | # usage 10 | 11 | ```python 12 | import os 13 | from litellm import longer_context_model_fallback_dict, ContextWindowExceededError, completion 14 | 15 | os.environ["HUGGINGFACE_API_KEY"] = "your-hf-token" # https://huggingface.co/docs/hub/security-tokens 16 | 17 | ## CREATE THE PROMPT 18 | prompt_prefix = 'def remove_non_ascii(s: str) -> str:\n """ ' 19 | prompt_suffix = "\n return result" 20 | 21 | ### set 
  to indicate the string before and after the part you want codellama to fill 
22 | prompt = f"
 {prompt_prefix} {prompt_suffix} "
23 | 
24 | messages = [{"content": prompt, "role": "user"}]
25 | model = "huggingface/codellama/CodeLlama-34b-Instruct-hf" # specify huggingface as the provider 'huggingface/'
26 | response = completion(model=model, messages=messages, max_tokens=500)
27 | ```
28 | 
29 | # output 
30 | ```python
31 | def remove_non_ascii(s: str) -> str:
32 |     """ Remove non-ASCII characters from a string.
33 | 
34 |     Args:
35 |         s (str): The string to remove non-ASCII characters from.
36 | 
37 |     Returns:
38 |         str: The string with non-ASCII characters removed.
39 |     """
40 |     result = ""
41 |     for c in s:
42 |         if ord(c) < 128:
43 |             result += c
44 |     return result
45 | ```


--------------------------------------------------------------------------------
/docs/my-website/docs/tutorials/installation.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | displayed_sidebar: tutorialSidebar
 3 | ---
 4 | 
 5 | # Set up environment
 6 | 
 7 | Let's get the necessary keys to set up our demo environment.
 8 | 
 9 | Every LLM provider needs API keys (e.g. `OPENAI_API_KEY`). You can get API keys from OpenAI, Cohere and AI21 **without a waitlist**.
10 | 
11 | Let's get them for our demo!
12 | 
13 | **OpenAI**: https://platform.openai.com/account/api-keys  
14 | **Cohere**: https://dashboard.cohere.com/welcome/login?redirect_uri=%2Fapi-keys  (no credit card required)  
15 | **AI21**: https://studio.ai21.com/account/api-key (no credit card required)
16 | 
17 | 
18 | 


--------------------------------------------------------------------------------
/docs/my-website/docs/tutorials/litellm_Test_Multiple_Providers_files/litellm_Test_Multiple_Providers_11_0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/docs/tutorials/litellm_Test_Multiple_Providers_files/litellm_Test_Multiple_Providers_11_0.png


--------------------------------------------------------------------------------
/docs/my-website/docs/tutorials/litellm_Test_Multiple_Providers_files/litellm_Test_Multiple_Providers_14_0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/docs/tutorials/litellm_Test_Multiple_Providers_files/litellm_Test_Multiple_Providers_14_0.png


--------------------------------------------------------------------------------
/docs/my-website/docs/tutorials/mock_completion.md:
--------------------------------------------------------------------------------
 1 | # Mock Completion Responses - Save Testing Costs
 2 | 
 3 | Trying to test making LLM Completion calls without calling the LLM APIs ? 
 4 | Pass `mock_response` to `litellm.completion` and litellm will directly return the response without neededing the call the LLM API and spend $$ 
 5 | 
 6 | ## Using `completion()` with `mock_response`
 7 | 
 8 | ```python
 9 | from litellm import completion 
10 | 
11 | model = "gpt-3.5-turbo"
12 | messages = [{"role":"user", "content":"Why is LiteLLM amazing?"}]
13 | 
14 | completion(model=model, messages=messages, mock_response="It's simple to use and easy to get started")
15 | ```
16 | 
17 | ## Building a pytest function using `completion`
18 | 
19 | ```python
20 | from litellm import completion
21 | import pytest
22 | 
23 | def test_completion_openai():
24 |     try:
25 |         response = completion(
26 |             model="gpt-3.5-turbo",
27 |             messages=[{"role":"user", "content":"Why is LiteLLM amazing?"}],
28 |             mock_response="LiteLLM is awesome"
29 |         )
30 |         # Add any assertions here to check the response
31 |         print(response)
32 |         print(response['choices'][0]['finish_reason'])
33 |     except Exception as e:
34 |         pytest.fail(f"Error occurred: {e}")
35 | ```
36 | 


--------------------------------------------------------------------------------
/docs/my-website/docs/tutorials/oobabooga.md:
--------------------------------------------------------------------------------
 1 | # Oobabooga Text Web API Tutorial
 2 | 
 3 | ### Install + Import LiteLLM 
 4 | ```python 
 5 | !pip install litellm
 6 | from litellm import completion 
 7 | import os
 8 | ```
 9 | 
10 | ### Call your oobabooga model
11 | Remember to set your api_base
12 | ```python
13 | response = completion(
14 |   model="oobabooga/WizardCoder-Python-7B-V1.0-GPTQ",
15 |   messages=[{ "content": "can you write a binary tree traversal preorder","role": "user"}], 
16 |   api_base="http://localhost:5000",
17 |   max_tokens=4000
18 | )
19 | ```
20 | 
21 | ### See your response 
22 | ```python 
23 | print(response)
24 | ```
25 | 
26 | Credits to [Shuai Shao](https://www.linkedin.com/in/shuai-sh/), for this tutorial. 


--------------------------------------------------------------------------------
/docs/my-website/docs/tutorials/provider_specific_params.md:
--------------------------------------------------------------------------------
 1 | ### Setting provider-specific Params
 2 | 
 3 | Goal: Set max tokens across OpenAI + Cohere
 4 | 
 5 | **1. via completion**
 6 | 
 7 | LiteLLM will automatically translate max_tokens to the naming convention followed by that specific model provider.
 8 | 
 9 | ```python
10 | from litellm import completion
11 | import os
12 | 
13 | ## set ENV variables 
14 | os.environ["OPENAI_API_KEY"] = "your-openai-key" 
15 | os.environ["COHERE_API_KEY"] = "your-cohere-key" 
16 | 
17 | messages = [{ "content": "Hello, how are you?","role": "user"}]
18 | 
19 | # openai call
20 | response = completion(model="gpt-3.5-turbo", messages=messages, max_tokens=100)
21 | 
22 | # cohere call
23 | response = completion(model="command-nightly", messages=messages, max_tokens=100)
24 | print(response)
25 | ```
26 | 
27 | **2. via provider-specific config**
28 | 
29 | For every provider on LiteLLM, we've gotten their specific params (following their naming conventions, etc.). You can just set it for that provider by pulling up that provider via `litellm.Config`. 
30 | 
31 | All provider configs are typed and have docstrings, so you should see them autocompleted for you in VSCode with an explanation of what it means. 
32 | 
33 | Here's an example of setting max tokens through provider configs. 
34 | 
35 | 


--------------------------------------------------------------------------------
/docs/my-website/docs/tutorials/text_completion.md:
--------------------------------------------------------------------------------
 1 | # Using Text Completion Format - with Completion()
 2 | 
 3 | If your prefer interfacing with the OpenAI Text Completion format this tutorial covers how to use LiteLLM in this format
 4 | ```python
 5 | response = openai.Completion.create(
 6 |     model="text-davinci-003",
 7 |     prompt='Write a tagline for a traditional bavarian tavern',
 8 |     temperature=0,
 9 |     max_tokens=100)
10 | ```
11 | 
12 | ## Using LiteLLM in the Text Completion format
13 | ### With gpt-3.5-turbo
14 | ```python
15 | from litellm import text_completion
16 | response = text_completion(
17 |     model="gpt-3.5-turbo",
18 |     prompt='Write a tagline for a traditional bavarian tavern',
19 |     temperature=0,
20 |     max_tokens=100)
21 | ```
22 | 
23 | ### With text-davinci-003
24 | ```python
25 | response = text_completion(
26 |     model="text-davinci-003",
27 |     prompt='Write a tagline for a traditional bavarian tavern',
28 |     temperature=0,
29 |     max_tokens=100)
30 | ```
31 | 
32 | ### With llama2
33 | ```python
34 | response = text_completion(
35 |     model="togethercomputer/llama-2-70b-chat",
36 |     prompt='Write a tagline for a traditional bavarian tavern',
37 |     temperature=0,
38 |     max_tokens=100)
39 | ```


--------------------------------------------------------------------------------
/docs/my-website/img/ab_test_code.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/ab_test_code.png


--------------------------------------------------------------------------------
/docs/my-website/img/ab_test_logs.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/ab_test_logs.png


--------------------------------------------------------------------------------
/docs/my-website/img/add_model.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/add_model.png


--------------------------------------------------------------------------------
/docs/my-website/img/alt_dashboard.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/alt_dashboard.png


--------------------------------------------------------------------------------
/docs/my-website/img/bench_llm.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/bench_llm.png


--------------------------------------------------------------------------------
/docs/my-website/img/cloud_run0.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/cloud_run0.png


--------------------------------------------------------------------------------
/docs/my-website/img/cloud_run1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/cloud_run1.png


--------------------------------------------------------------------------------
/docs/my-website/img/cloud_run2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/cloud_run2.png


--------------------------------------------------------------------------------
/docs/my-website/img/cloud_run3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/cloud_run3.png


--------------------------------------------------------------------------------
/docs/my-website/img/codellama_formatted_input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/codellama_formatted_input.png


--------------------------------------------------------------------------------
/docs/my-website/img/codellama_input.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/codellama_input.png


--------------------------------------------------------------------------------
/docs/my-website/img/compare_llms.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/compare_llms.png


--------------------------------------------------------------------------------
/docs/my-website/img/dash_output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/dash_output.png


--------------------------------------------------------------------------------
/docs/my-website/img/dashboard.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/dashboard.png


--------------------------------------------------------------------------------
/docs/my-website/img/dashboard_log.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/dashboard_log.png


--------------------------------------------------------------------------------
/docs/my-website/img/dashboard_log_row.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/dashboard_log_row.png


--------------------------------------------------------------------------------
/docs/my-website/img/favicon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/favicon.png


--------------------------------------------------------------------------------
/docs/my-website/img/hf_inference_endpoint.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/hf_inference_endpoint.png


--------------------------------------------------------------------------------
/docs/my-website/img/hosted_debugger_usage_page.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/hosted_debugger_usage_page.png


--------------------------------------------------------------------------------
/docs/my-website/img/langfuse.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/langfuse.png


--------------------------------------------------------------------------------
/docs/my-website/img/langsmith.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/langsmith.png


--------------------------------------------------------------------------------
/docs/my-website/img/lite_logs.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/lite_logs.png


--------------------------------------------------------------------------------
/docs/my-website/img/lite_logs2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/lite_logs2.png


--------------------------------------------------------------------------------
/docs/my-website/img/litellm_streamlit_playground.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/litellm_streamlit_playground.png


--------------------------------------------------------------------------------
/docs/my-website/img/promptlayer.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/promptlayer.png


--------------------------------------------------------------------------------
/docs/my-website/img/sagemaker_deploy.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/sagemaker_deploy.png


--------------------------------------------------------------------------------
/docs/my-website/img/sagemaker_domain.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/sagemaker_domain.png


--------------------------------------------------------------------------------
/docs/my-website/img/sagemaker_endpoint.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/sagemaker_endpoint.png


--------------------------------------------------------------------------------
/docs/my-website/img/sagemaker_jumpstart.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/sagemaker_jumpstart.png


--------------------------------------------------------------------------------
/docs/my-website/img/slack.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/slack.png


--------------------------------------------------------------------------------
/docs/my-website/img/swagger.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/swagger.png


--------------------------------------------------------------------------------
/docs/my-website/img/term_output.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/term_output.png


--------------------------------------------------------------------------------
/docs/my-website/img/test_python_server_1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/test_python_server_1.png


--------------------------------------------------------------------------------
/docs/my-website/img/test_python_server_2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/test_python_server_2.png


--------------------------------------------------------------------------------
/docs/my-website/img/traceloop_dash.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/traceloop_dash.png


--------------------------------------------------------------------------------
/docs/my-website/img/wandb.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/img/wandb.png


--------------------------------------------------------------------------------
/docs/my-website/index.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | slug: welcome
 3 | title: Welcome
 4 | authors: [slorber, yangshun]
 5 | tags: [facebook, hello, docusaurus]
 6 | ---
 7 | 
 8 | [Docusaurus blogging features](https://docusaurus.io/docs/blog) are powered by the [blog plugin](https://docusaurus.io/docs/api/plugins/@docusaurus/plugin-content-blog).
 9 | 
10 | Simply add Markdown files (or folders) to the `blog` directory.
11 | 
12 | Regular blog authors can be added to `authors.yml`.
13 | 
14 | The blog post date can be extracted from filenames, such as:
15 | 
16 | - `2019-05-30-welcome.md`
17 | - `2019-05-30-welcome/index.md`
18 | 
19 | A blog post folder can be convenient to co-locate blog post images:
20 | 
21 | ![Docusaurus Plushie](./docusaurus-plushie-banner.jpeg)
22 | 
23 | The blog supports tags as well!
24 | 
25 | **And if you don't want a blog**: just delete this directory, and use `blog: false` in your Docusaurus config.


--------------------------------------------------------------------------------
/docs/my-website/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "my-website",
 3 |   "version": "0.0.0",
 4 |   "private": true,
 5 |   "scripts": {
 6 |     "docusaurus": "docusaurus",
 7 |     "start": "docusaurus start",
 8 |     "build": "docusaurus build",
 9 |     "swizzle": "docusaurus swizzle",
10 |     "deploy": "docusaurus deploy",
11 |     "clear": "docusaurus clear",
12 |     "serve": "docusaurus serve",
13 |     "write-translations": "docusaurus write-translations",
14 |     "write-heading-ids": "docusaurus write-heading-ids"
15 |   },
16 |   "dependencies": {
17 |     "@docusaurus/core": "2.4.1",
18 |     "@docusaurus/plugin-google-gtag": "^2.4.1",
19 |     "@docusaurus/plugin-ideal-image": "^2.4.1",
20 |     "@docusaurus/preset-classic": "2.4.1",
21 |     "@mdx-js/react": "^1.6.22",
22 |     "clsx": "^1.2.1",
23 |     "docusaurus-lunr-search": "^2.4.1",
24 |     "prism-react-renderer": "^1.3.5",
25 |     "react": "^17.0.2",
26 |     "react-dom": "^17.0.2",
27 |     "sharp": "^0.32.5",
28 |     "uuid": "^9.0.1"
29 |   },
30 |   "devDependencies": {
31 |     "@docusaurus/module-type-aliases": "2.4.1"
32 |   },
33 |   "browserslist": {
34 |     "production": [
35 |       ">0.5%",
36 |       "not dead",
37 |       "not op_mini all"
38 |     ],
39 |     "development": [
40 |       "last 1 chrome version",
41 |       "last 1 firefox version",
42 |       "last 1 safari version"
43 |     ]
44 |   },
45 |   "engines": {
46 |     "node": ">=16.14"
47 |   }
48 | }
49 | 


--------------------------------------------------------------------------------
/docs/my-website/src/components/CrispChat.js:
--------------------------------------------------------------------------------
 1 | import React, { useEffect } from 'react';
 2 | 
 3 | const CrispChat = () => {
 4 |     useEffect(() => {
 5 |         window.$crisp = [];
 6 |         window.CRISP_WEBSITE_ID = "be07a4d6-dba0-4df7-961d-9302c86b7ebc";
 7 | 
 8 |         const d = document;
 9 |         const s = d.createElement("script");
10 |         s.src = "https://client.crisp.chat/l.js";
11 |         s.async = 1;
12 |         document.getElementsByTagName("head")[0].appendChild(s);
13 |     }, [])
14 |   
15 |     return null;
16 | };
17 | 
18 | export default CrispChat;


--------------------------------------------------------------------------------
/docs/my-website/src/components/HomepageFeatures/index.js:
--------------------------------------------------------------------------------
 1 | import React from 'react';
 2 | import clsx from 'clsx';
 3 | import styles from './styles.module.css';
 4 | 
 5 | const FeatureList = [
 6 |   {
 7 |     title: 'Easy to Use',
 8 |     Svg: require('@site/static/img/undraw_docusaurus_mountain.svg').default,
 9 |     description: (
10 |       <>
11 |         Docusaurus was designed from the ground up to be easily installed and
12 |         used to get your website up and running quickly.
13 |       
14 |     ),
15 |   },
16 |   {
17 |     title: 'Focus on What Matters',
18 |     Svg: require('@site/static/img/undraw_docusaurus_tree.svg').default,
19 |     description: (
20 |       <>
21 |         Docusaurus lets you focus on your docs, and we'll do the chores. Go
22 |         ahead and move your docs into the docs directory.
23 |       
24 |     ),
25 |   },
26 |   {
27 |     title: 'Powered by React',
28 |     Svg: require('@site/static/img/undraw_docusaurus_react.svg').default,
29 |     description: (
30 |       <>
31 |         Extend or customize your website layout by reusing React. Docusaurus can
32 |         be extended while reusing the same header and footer.
33 |       
34 |     ),
35 |   },
36 | ];
37 | 
38 | function Feature({Svg, title, description}) {
39 |   return (
40 |     

41 |       

42 |         
43 |       

44 |       

45 |         
{title}

46 |         
{description}

47 |       

48 |     

49 |   );
50 | }
51 | 
52 | export default function HomepageFeatures() {
53 |   return (
54 |     

55 |       

56 |         

57 |           {FeatureList.map((props, idx) => (
58 |             
59 |           ))}
60 |         

61 |       

62 |     

63 |   );
64 | }
65 | 


--------------------------------------------------------------------------------
/docs/my-website/src/components/HomepageFeatures/styles.module.css:
--------------------------------------------------------------------------------
 1 | .features {
 2 |   display: flex;
 3 |   align-items: center;
 4 |   padding: 2rem 0;
 5 |   width: 100%;
 6 | }
 7 | 
 8 | .featureSvg {
 9 |   height: 200px;
10 |   width: 200px;
11 | }
12 | 


--------------------------------------------------------------------------------
/docs/my-website/src/components/QuickStart.js:
--------------------------------------------------------------------------------
 1 | import React, { useState, useEffect } from 'react';
 2 | 
 3 | const QuickStartCodeBlock = ({ token }) => {
 4 |     return (
 5 |       
 6 |         {`
 7 |         from litellm import completion
 8 |         import os
 9 |   
10 |         ## set ENV variables
11 |         os.environ["OPENAI_API_KEY"] = "${token}"
12 |         os.environ["COHERE_API_KEY"] = "${token}"
13 |   
14 |         messages = [{ "content": "Hello, how are you?","role": "user"}]
15 |   
16 |         # openai call
17 |         response = completion(model="gpt-3.5-turbo", messages=messages)
18 |   
19 |         # cohere call
20 |         response = completion("command-nightly", messages)
21 |         `}
22 |       

23 |     );
24 |   };
25 |   
26 |   const QuickStart = () => {
27 |     const [token, setToken] = useState(null);
28 |   
29 |     useEffect(() => {
30 |       const generateToken = async () => {
31 |         try {
32 |           const response = await fetch('https://proxy.litellm.ai/key/new', {
33 |             method: 'POST',
34 |             headers: {
35 |               'Content-Type': 'application/json',
36 |               'Authorization': 'Bearer sk-liteplayground',
37 |             },
38 |             body: JSON.stringify({'total_budget': 100})
39 |           });
40 |           
41 |           if (!response.ok) {
42 |             throw new Error('Network response was not ok');
43 |           }
44 |           
45 |           const data = await response.json();
46 |   
47 |           setToken(`${data.api_key}`);
48 |       } catch (error) {
49 |         console.error('Failed to fetch new token: ', error);
50 |       }
51 |     };
52 |   
53 |     generateToken();
54 |   }, []);
55 |   
56 |   return (
57 |     

58 |       
59 |     

60 |   );
61 |   }
62 | 
63 |   export default QuickStart;


--------------------------------------------------------------------------------
/docs/my-website/src/components/TokenGen.js:
--------------------------------------------------------------------------------
 1 | import React, { useState, useEffect } from 'react';
 2 | 
 3 | const CodeBlock = ({ token }) => {
 4 |   const codeWithToken = `${token}`;
 5 | 
 6 |   return (
 7 |     
 8 |       {token ? codeWithToken : ""}
 9 |     

10 |   );
11 | };
12 | 
13 | const TokenGen = () => {
14 |   const [token, setToken] = useState(null);
15 | 
16 |   useEffect(() => {
17 |     const generateToken = async () => {
18 |       try {
19 |         const response = await fetch('https://proxy.litellm.ai/key/new', {
20 |           method: 'POST',
21 |           headers: {
22 |             'Content-Type': 'application/json',
23 |             'Authorization': 'Bearer sk-liteplayground',
24 |           },
25 |           body: JSON.stringify({'total_budget': 100})
26 |         });
27 |         
28 |         if (!response.ok) {
29 |           throw new Error('Network response was not ok');
30 |         }
31 |         
32 |         const data = await response.json();
33 | 
34 |         setToken(`${data.api_key}`);
35 |     } catch (error) {
36 |       console.error('Failed to fetch new token: ', error);
37 |     }
38 |   };
39 | 
40 |   generateToken();
41 | }, []);
42 | 
43 | return (
44 |   

45 |     
46 |   

47 | );
48 | };
49 | 
50 | export default TokenGen;
51 | 


--------------------------------------------------------------------------------
/docs/my-website/src/components/queryParamReader.js:
--------------------------------------------------------------------------------
 1 | import React, { useState, useEffect } from 'react';
 2 | 
 3 | const CodeBlock = ({ token }) => {
 4 |   const codeWithToken = `
 5 | import os
 6 | from litellm import completion
 7 | 
 8 | # set ENV variables 
 9 | os.environ["LITELLM_TOKEN"] = '${token}'
10 | 
11 | messages = [{ "content": "Hello, how are you?","role": "user"}]
12 | 
13 | # openai call
14 | response = completion(model="gpt-3.5-turbo", messages=messages)
15 | 
16 | # cohere call
17 | response = completion("command-nightly", messages)
18 | `;
19 | 
20 |   const codeWithoutToken = `
21 | from litellm import completion
22 | 
23 | ## set ENV variables
24 | os.environ["OPENAI_API_KEY"] = "openai key"
25 | os.environ["COHERE_API_KEY"] = "cohere key"
26 | 
27 | 
28 | messages = [{ "content": "Hello, how are you?","role": "user"}]
29 | 
30 | # openai call
31 | response = completion(model="gpt-3.5-turbo", messages=messages)
32 | 
33 | # cohere call
34 | response = completion("command-nightly", messages)
35 | `;
36 |   return (
37 |     
38 |         {console.log("token: ", token)}
39 |       {token ? codeWithToken : codeWithoutToken}
40 |     

41 |   )
42 | }
43 | 
44 | const QueryParamReader = () => {
45 |   const [token, setToken] = useState(null);
46 | 
47 |   useEffect(() => {
48 |     const urlParams = new URLSearchParams(window.location.search);
49 |     console.log("urlParams: ", urlParams)
50 |     const token = urlParams.get('token');
51 |     setToken(token);
52 |   }, []);
53 | 
54 |   return (
55 |     

56 |       
57 |     

58 |   );
59 | }
60 | 
61 | export default QueryParamReader;


--------------------------------------------------------------------------------
/docs/my-website/src/components/queryParamToken.js:
--------------------------------------------------------------------------------
 1 | import React, { useState, useEffect } from 'react';
 2 | 
 3 | const QueryParamToken = () => {
 4 |   const [token, setToken] = useState(null);
 5 | 
 6 |   useEffect(() => {
 7 |     const urlParams = new URLSearchParams(window.location.search);
 8 |     const token = urlParams.get('token');
 9 |     setToken(token);
10 |   }, []);
11 | 
12 |   return (
13 |     
14 |       {token ? admin.litellm.ai : ""}
15 |     
16 |   );
17 | }
18 | 
19 | export default QueryParamToken;


--------------------------------------------------------------------------------
/docs/my-website/src/css/custom.css:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Any CSS included here will be global. The classic template
 3 |  * bundles Infima by default. Infima is a CSS framework designed to
 4 |  * work well for content-centric websites.
 5 |  */
 6 | 
 7 | /* You can override the default Infima variables here. */
 8 | :root {
 9 |   --ifm-color-primary: #2e8555;
10 |   --ifm-color-primary-dark: #29784c;
11 |   --ifm-color-primary-darker: #277148;
12 |   --ifm-color-primary-darkest: #205d3b;
13 |   --ifm-color-primary-light: #33925d;
14 |   --ifm-color-primary-lighter: #359962;
15 |   --ifm-color-primary-lightest: #3cad6e;
16 |   --ifm-code-font-size: 95%;
17 |   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1);
18 | }
19 | 
20 | /* For readability concerns, you should choose a lighter palette in dark mode. */
21 | [data-theme='dark'] {
22 |   --ifm-color-primary: #25c2a0;
23 |   --ifm-color-primary-dark: #21af90;
24 |   --ifm-color-primary-darker: #1fa588;
25 |   --ifm-color-primary-darkest: #1a8870;
26 |   --ifm-color-primary-light: #29d5b0;
27 |   --ifm-color-primary-lighter: #32d8b4;
28 |   --ifm-color-primary-lightest: #4fddbf;
29 |   --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
30 | }
31 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages-fake/index.js:
--------------------------------------------------------------------------------
 1 | import React from 'react';
 2 | import clsx from 'clsx';
 3 | import Link from '@docusaurus/Link';
 4 | import useDocusaurusContext from '@docusaurus/useDocusaurusContext';
 5 | import Layout from '@theme/Layout';
 6 | import HomepageFeatures from '@site/src/components/HomepageFeatures';
 7 | 
 8 | import styles from './index.module.css';
 9 | 
10 | function HomepageHeader() {
11 |   const {siteConfig} = useDocusaurusContext();
12 |   return (
13 |     

14 |       

15 |         
{siteConfig.title}

16 |         
{siteConfig.tagline}

17 |         

18 |           
21 |             Docusaurus Tutorial - 5min ⏱️
22 |           
23 |         

24 |       

25 |     

26 |   );
27 | }
28 | 
29 | export default function Home() {
30 |   const {siteConfig} = useDocusaurusContext();
31 |   return (
32 |     
35 |       
36 |       

37 |         
38 |       

39 |     
40 |   );
41 | }
42 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages-fake/index.module.css:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * CSS files with the .module.css suffix will be treated as CSS modules
 3 |  * and scoped locally.
 4 |  */
 5 | 
 6 | .heroBanner {
 7 |   padding: 4rem 0;
 8 |   text-align: center;
 9 |   position: relative;
10 |   overflow: hidden;
11 | }
12 | 
13 | @media screen and (max-width: 996px) {
14 |   .heroBanner {
15 |     padding: 2rem;
16 |   }
17 | }
18 | 
19 | .buttons {
20 |   display: flex;
21 |   align-items: center;
22 |   justify-content: center;
23 | }
24 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages-fake/markdown-page.md:
--------------------------------------------------------------------------------
1 | ---
2 | title: Markdown page example
3 | ---
4 | 
5 | # Markdown page example
6 | 
7 | You don't need React to write simple standalone pages.
8 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/completion/output.md:
--------------------------------------------------------------------------------
 1 | # Completion Function - completion()
 2 | Here's the exact json output you can expect from a litellm `completion` call:
 3 | 
 4 | ```python 
 5 | {'choices': [{'finish_reason': 'stop',
 6 |    'index': 0,
 7 |    'message': {'role': 'assistant',
 8 |     'content': " I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic."}}],
 9 |  'created': 1691429984.3852863,
10 |  'model': 'claude-instant-1',
11 |  'usage': {'prompt_tokens': 18, 'completion_tokens': 23, 'total_tokens': 41}}
12 | ```


--------------------------------------------------------------------------------
/docs/my-website/src/pages/contact.md:
--------------------------------------------------------------------------------
1 | # Contact Us
2 | 
3 | [![](https://dcbadge.vercel.app/api/server/wuPM9dRgDw)](https://discord.gg/wuPM9dRgDw)
4 | 
5 | * [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
6 | * Contact us at ishaan@berri.ai / krrish@berri.ai
7 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/contributing.md:
--------------------------------------------------------------------------------
 1 | # Contributing to Documentation
 2 | 
 3 | Clone litellm 
 4 | ```
 5 | git clone https://github.com/BerriAI/litellm.git
 6 | ```
 7 | 
 8 | ### Local setup for locally running docs
 9 | 
10 | #### Installation
11 | ```
12 | pip install mkdocs
13 | ```
14 | 
15 | #### Locally Serving Docs
16 | ```
17 | mkdocs serve
18 | ```
19 | If you see `command not found: mkdocs` try running the following
20 | ```
21 | python3 -m mkdocs serve
22 | ```
23 | 
24 | This command builds your Markdown files into HTML and starts a development server to browse your documentation. Open up [http://127.0.0.1:8000/](http://127.0.0.1:8000/) in your web browser to see your documentation. You can make changes to your Markdown files and your docs will automatically rebuild.
25 | 
26 | [Full tutorial here](https://docs.readthedocs.io/en/stable/intro/getting-started-with-mkdocs.html)
27 | 
28 | ### Making changes to Docs
29 | - All the docs are placed under the `docs` directory
30 | - If you are adding a new `.md` file or editing the hierarchy edit `mkdocs.yml` in the root of the project
31 | - After testing your changes, make a change to the `main` branch of [github.com/BerriAI/litellm](https://github.com/BerriAI/litellm)
32 | 
33 | 
34 | 
35 | 
36 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/embedding/supported_embedding.md:
--------------------------------------------------------------------------------
1 | # Embedding Models
2 | 
3 | | Model Name           | Function Call                               | Required OS Variables                |
4 | |----------------------|---------------------------------------------|--------------------------------------|
5 | | text-embedding-ada-002 | `embedding('text-embedding-ada-002', input)` | `os.environ['OPENAI_API_KEY']`       |


--------------------------------------------------------------------------------
/docs/my-website/src/pages/intro.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | sidebar_position: 1
 3 | ---
 4 | 
 5 | # Tutorial Intro
 6 | 
 7 | Let's discover **Docusaurus in less than 5 minutes**.
 8 | 
 9 | ## Getting Started
10 | 
11 | Get started by **creating a new site**.
12 | 
13 | Or **try Docusaurus immediately** with **[docusaurus.new](https://docusaurus.new)**.
14 | 
15 | ### What you'll need
16 | 
17 | - [Node.js](https://nodejs.org/en/download/) version 16.14 or above:
18 |   - When installing Node.js, you are recommended to check all checkboxes related to dependencies.
19 | 
20 | ## Generate a new site
21 | 
22 | Generate a new Docusaurus site using the **classic template**.
23 | 
24 | The classic template will automatically be added to your project after you run the command:
25 | 
26 | ```bash
27 | npm init docusaurus@latest my-website classic
28 | ```
29 | 
30 | You can type this command into Command Prompt, Powershell, Terminal, or any other integrated terminal of your code editor.
31 | 
32 | The command also installs all necessary dependencies you need to run Docusaurus.
33 | 
34 | ## Start your site
35 | 
36 | Run the development server:
37 | 
38 | ```bash
39 | cd my-website
40 | npm run start
41 | ```
42 | 
43 | The `cd` command changes the directory you're working with. In order to work with your newly created Docusaurus site, you'll need to navigate the terminal there.
44 | 
45 | The `npm run start` command builds your website locally and serves it through a development server, ready for you to view at http://localhost:3000/.
46 | 
47 | Open `docs/intro.md` (this page) and edit some lines: the site **reloads automatically** and displays your changes.
48 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/observability/callbacks.md:
--------------------------------------------------------------------------------
 1 | # Callbacks
 2 | 
 3 | ## Use Callbacks to send Output Data to Posthog, Sentry etc
 4 | 
 5 | liteLLM provides `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
 6 | 
 7 | liteLLM supports:
 8 | 
 9 | - [LLMonitor](https://llmonitor.com/docs)
10 | - [Helicone](https://docs.helicone.ai/introduction)
11 | - [Sentry](https://docs.sentry.io/platforms/python/)
12 | - [PostHog](https://posthog.com/docs/libraries/python)
13 | - [Slack](https://slack.dev/bolt-python/concepts)
14 | 
15 | ### Quick Start
16 | 
17 | ```python
18 | from litellm import completion
19 | 
20 | # set callbacks
21 | litellm.success_callback=["posthog", "helicone", "llmonitor"]
22 | litellm.failure_callback=["sentry", "llmonitor"]
23 | 
24 | ## set env variables
25 | os.environ['SENTRY_API_URL'], os.environ['SENTRY_API_TRACE_RATE']= ""
26 | os.environ['POSTHOG_API_KEY'], os.environ['POSTHOG_API_URL'] = "api-key", "api-url"
27 | os.environ["HELICONE_API_KEY"] = ""
28 | 
29 | response = completion(model="gpt-3.5-turbo", messages=messages)
30 | ```
31 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/observability/helicone_integration.md:
--------------------------------------------------------------------------------
 1 | # Helicone Tutorial 
 2 | [Helicone](https://helicone.ai/) is an open source observability platform that proxies your OpenAI traffic and provides you key insights into your spend, latency and usage.
 3 | 
 4 | ## Use Helicone to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM)
 5 | liteLLM provides `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses. 
 6 | 
 7 | In this case, we want to log requests to Helicone when a request succeeds. 
 8 | 
 9 | ### Approach 1: Use Callbacks 
10 | Use just 1 line of code, to instantly log your responses **across all providers** with helicone: 
11 | ```
12 | litellm.success_callback=["helicone"]
13 | ```
14 | 
15 | Complete code
16 | ```python
17 | from litellm import completion
18 | 
19 | ## set env variables
20 | os.environ["HELICONE_API_KEY"] = "your-helicone-key" 
21 | os.environ["OPENAI_API_KEY"], os.environ["COHERE_API_KEY"] = "", ""
22 | 
23 | # set callbacks
24 | litellm.success_callback=["helicone"]
25 | 
26 | #openai call
27 | response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) 
28 | 
29 | #cohere call
30 | response = completion(model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}]) 
31 | ```
32 | 
33 | ### Approach 2: [OpenAI + Azure only] Use Helicone as a proxy
34 | Helicone provides advanced functionality like caching, etc. Helicone currently supports this for Azure and OpenAI.
35 | 
36 | If you want to use Helicone to proxy your OpenAI/Azure requests, then you can - 
37 | 
38 | - Set helicone as your base url via: `litellm.api_url` 
39 | - Pass in helicone request headers via: `litellm.headers` 
40 | 
41 | Complete Code
42 | ```
43 | import litellm
44 | from litellm import completion
45 | 
46 | litellm.api_base = "https://oai.hconeai.com/v1"
47 | litellm.headers = {"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}"}
48 | 
49 | response = litellm.completion(
50 |     model="gpt-3.5-turbo",
51 |     messages=[{"role": "user", "content": "how does a court case get to the Supreme Court?"}]
52 | )
53 | 
54 | print(response)
55 | ```
56 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/secret.md:
--------------------------------------------------------------------------------
 1 | # Secret Managers
 2 | liteLLM reads secrets from yoour secret manager, .env file 
 3 | 
 4 | - [Infisical Secret Manager](#infisical-secret-manager)
 5 | - [.env Files](#env-files)
 6 | 
 7 | For expected format of secrets see [supported LLM models](https://litellm.readthedocs.io/en/latest/supported)
 8 | 
 9 | ## Infisical Secret Manager
10 | Integrates with [Infisical's Secret Manager](https://infisical.com/) for secure storage and retrieval of API keys and sensitive data.
11 | 
12 | ### Usage
13 | liteLLM manages reading in your LLM API secrets/env variables from Infisical for you
14 | 
15 | ```
16 | import litellm
17 | from infisical import InfisicalClient
18 | 
19 | litellm.secret_manager = InfisicalClient(token="your-token")
20 | 
21 | messages = [
22 |     {"role": "system", "content": "You are a helpful assistant."},
23 |     {"role": "user", "content": "What's the weather like today?"},
24 | ]
25 | 
26 | response = litellm.completion(model="gpt-3.5-turbo", messages=messages)
27 | 
28 | print(response)
29 | ```
30 | 
31 | 
32 | ## .env Files
33 | If no secret manager client is specified, Litellm automatically uses the `.env` file to manage sensitive data.
34 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/stream.md:
--------------------------------------------------------------------------------
 1 | # Streaming Responses & Async Completion
 2 | 
 3 | - [Streaming Responses](#streaming-responses)
 4 | - [Async Completion](#async-completion)
 5 | 
 6 | ## Streaming Responses
 7 | LiteLLM supports streaming the model response back by passing `stream=True` as an argument to the completion function
 8 | ### Usage
 9 | ```python
10 | response = completion(model="gpt-3.5-turbo", messages=messages, stream=True)
11 | for chunk in response:
12 |     print(chunk['choices'][0]['delta'])
13 | 
14 | ```
15 | 
16 | ## Async Completion
17 | Asynchronous Completion with LiteLLM
18 | LiteLLM provides an asynchronous version of the completion function called `acompletion`
19 | ### Usage
20 | ```
21 | from litellm import acompletion
22 | import asyncio
23 | 
24 | async def test_get_response():
25 |     user_message = "Hello, how are you?"
26 |     messages = [{"content": user_message, "role": "user"}]
27 |     response = await acompletion(model="gpt-3.5-turbo", messages=messages)
28 |     return response
29 | 
30 | response = asyncio.run(test_get_response())
31 | print(response)
32 | 
33 | ```


--------------------------------------------------------------------------------
/docs/my-website/src/pages/token_usage.md:
--------------------------------------------------------------------------------
 1 | # Token Usage
 2 | By default LiteLLM returns token usage in all completion requests ([See here](https://litellm.readthedocs.io/en/latest/output/))
 3 | 
 4 | However, we also expose 3 public helper functions to calculate token usage across providers:
 5 | 
 6 | - `token_counter`: This returns the number of tokens for a given input - it uses the tokenizer based on the model, and defaults to tiktoken if no model-specific tokenizer is available. 
 7 | 
 8 | - `cost_per_token`: This returns the cost (in USD) for prompt (input) and completion (output) tokens. It utilizes our model_cost map which can be found in `__init__.py` and also as a [community resource](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
 9 | 
10 | - `completion_cost`: This returns the overall cost (in USD) for a given LLM API Call. It combines `token_counter` and `cost_per_token` to return the cost for that query (counting both cost of input and output). 
11 | 
12 | ## Example Usage 
13 | 
14 | 1. `token_counter`
15 | 
16 | ```python
17 | from litellm import token_counter
18 | 
19 | messages = [{"user": "role", "content": "Hey, how's it going"}]
20 | print(token_counter(model="gpt-3.5-turbo", messages=messages))
21 | ```
22 | 
23 | 2. `cost_per_token`
24 | 
25 | ```python
26 | from litellm import cost_per_token
27 | 
28 | prompt_tokens =  5
29 | completion_tokens = 10
30 | prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens))
31 | 
32 | print(prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar)
33 | ```
34 | 
35 | 3. `completion_cost`
36 | 
37 | ```python
38 | from litellm import completion_cost
39 | 
40 | prompt = "Hey, how's it going"
41 | completion = "Hi, I'm gpt - I am doing well"
42 | cost_of_query = completion_cost(model="gpt-3.5-turbo", prompt=prompt, completion=completion))
43 | 
44 | print(cost_of_query)
45 | ```
46 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/troubleshoot.md:
--------------------------------------------------------------------------------
 1 | # Troubleshooting 
 2 | 
 3 | ## Stable Version
 4 | 
 5 | If you're running into problems with installation / Usage 
 6 | Use the stable version of litellm 
 7 | 
 8 | ```
 9 | pip install litellm==0.1.345
10 | ```
11 | 
12 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-basics/_category_.json:
--------------------------------------------------------------------------------
1 | {
2 |   "label": "Tutorial - Basics",
3 |   "position": 2,
4 |   "link": {
5 |     "type": "generated-index",
6 |     "description": "5 minutes to learn the most important Docusaurus concepts."
7 |   }
8 | }
9 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-basics/congratulations.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | sidebar_position: 6
 3 | ---
 4 | 
 5 | # Congratulations!
 6 | 
 7 | You have just learned the **basics of Docusaurus** and made some changes to the **initial template**.
 8 | 
 9 | Docusaurus has **much more to offer**!
10 | 
11 | Have **5 more minutes**? Take a look at **[versioning](../tutorial-extras/manage-docs-versions.md)** and **[i18n](../tutorial-extras/translate-your-site.md)**.
12 | 
13 | Anything **unclear** or **buggy** in this tutorial? [Please report it!](https://github.com/facebook/docusaurus/discussions/4610)
14 | 
15 | ## What's next?
16 | 
17 | - Read the [official documentation](https://docusaurus.io/)
18 | - Modify your site configuration with [`docusaurus.config.js`](https://docusaurus.io/docs/api/docusaurus-config)
19 | - Add navbar and footer items with [`themeConfig`](https://docusaurus.io/docs/api/themes/configuration)
20 | - Add a custom [Design and Layout](https://docusaurus.io/docs/styling-layout)
21 | - Add a [search bar](https://docusaurus.io/docs/search)
22 | - Find inspirations in the [Docusaurus showcase](https://docusaurus.io/showcase)
23 | - Get involved in the [Docusaurus Community](https://docusaurus.io/community/support)
24 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | sidebar_position: 3
 3 | ---
 4 | 
 5 | # Create a Blog Post
 6 | 
 7 | Docusaurus creates a **page for each blog post**, but also a **blog index page**, a **tag system**, an **RSS** feed...
 8 | 
 9 | ## Create your first Post
10 | 
11 | Create a file at `blog/2021-02-28-greetings.md`:
12 | 
13 | ```md title="blog/2021-02-28-greetings.md"
14 | ---
15 | slug: greetings
16 | title: Greetings!
17 | authors:
18 |   - name: Joel Marcey
19 |     title: Co-creator of Docusaurus 1
20 |     url: https://github.com/JoelMarcey
21 |     image_url: https://github.com/JoelMarcey.png
22 |   - name: Sébastien Lorber
23 |     title: Docusaurus maintainer
24 |     url: https://sebastienlorber.com
25 |     image_url: https://github.com/slorber.png
26 | tags: [greetings]
27 | ---
28 | 
29 | Congratulations, you have made your first post!
30 | 
31 | Feel free to play around and edit this post as much you like.
32 | ```
33 | 
34 | A new blog post is now available at [http://localhost:3000/blog/greetings](http://localhost:3000/blog/greetings).
35 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-basics/create-a-document.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | sidebar_position: 2
 3 | ---
 4 | 
 5 | # Create a Document
 6 | 
 7 | Documents are **groups of pages** connected through:
 8 | 
 9 | - a **sidebar**
10 | - **previous/next navigation**
11 | - **versioning**
12 | 
13 | ## Create your first Doc
14 | 
15 | Create a Markdown file at `docs/hello.md`:
16 | 
17 | ```md title="docs/hello.md"
18 | # Hello
19 | 
20 | This is my **first Docusaurus document**!
21 | ```
22 | 
23 | A new document is now available at [http://localhost:3000/docs/hello](http://localhost:3000/docs/hello).
24 | 
25 | ## Configure the Sidebar
26 | 
27 | Docusaurus automatically **creates a sidebar** from the `docs` folder.
28 | 
29 | Add metadata to customize the sidebar label and position:
30 | 
31 | ```md title="docs/hello.md" {1-4}
32 | ---
33 | sidebar_label: 'Hi!'
34 | sidebar_position: 3
35 | ---
36 | 
37 | # Hello
38 | 
39 | This is my **first Docusaurus document**!
40 | ```
41 | 
42 | It is also possible to create your sidebar explicitly in `sidebars.js`:
43 | 
44 | ```js title="sidebars.js"
45 | module.exports = {
46 |   tutorialSidebar: [
47 |     'intro',
48 |     // highlight-next-line
49 |     'hello',
50 |     {
51 |       type: 'category',
52 |       label: 'Tutorial',
53 |       items: ['tutorial-basics/create-a-document'],
54 |     },
55 |   ],
56 | };
57 | ```
58 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-basics/create-a-page.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | sidebar_position: 1
 3 | ---
 4 | 
 5 | # Create a Page
 6 | 
 7 | Add **Markdown or React** files to `src/pages` to create a **standalone page**:
 8 | 
 9 | - `src/pages/index.js` → `localhost:3000/`
10 | - `src/pages/foo.md` → `localhost:3000/foo`
11 | - `src/pages/foo/bar.js` → `localhost:3000/foo/bar`
12 | 
13 | ## Create your first React Page
14 | 
15 | Create a file at `src/pages/my-react-page.js`:
16 | 
17 | ```jsx title="src/pages/my-react-page.js"
18 | import React from 'react';
19 | import Layout from '@theme/Layout';
20 | 
21 | export default function MyReactPage() {
22 |   return (
23 |     
24 |       
My React page

25 |       
This is a React page

26 |     
27 |   );
28 | }
29 | ```
30 | 
31 | A new page is now available at [http://localhost:3000/my-react-page](http://localhost:3000/my-react-page).
32 | 
33 | ## Create your first Markdown Page
34 | 
35 | Create a file at `src/pages/my-markdown-page.md`:
36 | 
37 | ```mdx title="src/pages/my-markdown-page.md"
38 | # My Markdown page
39 | 
40 | This is a Markdown page
41 | ```
42 | 
43 | A new page is now available at [http://localhost:3000/my-markdown-page](http://localhost:3000/my-markdown-page).
44 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-basics/deploy-your-site.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | sidebar_position: 5
 3 | ---
 4 | 
 5 | # Deploy your site
 6 | 
 7 | Docusaurus is a **static-site-generator** (also called **[Jamstack](https://jamstack.org/)**).
 8 | 
 9 | It builds your site as simple **static HTML, JavaScript and CSS files**.
10 | 
11 | ## Build your site
12 | 
13 | Build your site **for production**:
14 | 
15 | ```bash
16 | npm run build
17 | ```
18 | 
19 | The static files are generated in the `build` folder.
20 | 
21 | ## Deploy your site
22 | 
23 | Test your production build locally:
24 | 
25 | ```bash
26 | npm run serve
27 | ```
28 | 
29 | The `build` folder is now served at [http://localhost:3000/](http://localhost:3000/).
30 | 
31 | You can now deploy the `build` folder **almost anywhere** easily, **for free** or very small cost (read the **[Deployment Guide](https://docusaurus.io/docs/deployment)**).
32 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-extras/_category_.json:
--------------------------------------------------------------------------------
1 | {
2 |   "label": "Tutorial - Extras",
3 |   "position": 3,
4 |   "link": {
5 |     "type": "generated-index"
6 |   }
7 | }
8 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-extras/img/docsVersionDropdown.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/src/pages/tutorial-extras/img/docsVersionDropdown.png


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-extras/img/localeDropdown.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/src/pages/tutorial-extras/img/localeDropdown.png


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | sidebar_position: 1
 3 | ---
 4 | 
 5 | # Manage Docs Versions
 6 | 
 7 | Docusaurus can manage multiple versions of your docs.
 8 | 
 9 | ## Create a docs version
10 | 
11 | Release a version 1.0 of your project:
12 | 
13 | ```bash
14 | npm run docusaurus docs:version 1.0
15 | ```
16 | 
17 | The `docs` folder is copied into `versioned_docs/version-1.0` and `versions.json` is created.
18 | 
19 | Your docs now have 2 versions:
20 | 
21 | - `1.0` at `http://localhost:3000/docs/` for the version 1.0 docs
22 | - `current` at `http://localhost:3000/docs/next/` for the **upcoming, unreleased docs**
23 | 
24 | ## Add a Version Dropdown
25 | 
26 | To navigate seamlessly across versions, add a version dropdown.
27 | 
28 | Modify the `docusaurus.config.js` file:
29 | 
30 | ```js title="docusaurus.config.js"
31 | module.exports = {
32 |   themeConfig: {
33 |     navbar: {
34 |       items: [
35 |         // highlight-start
36 |         {
37 |           type: 'docsVersionDropdown',
38 |         },
39 |         // highlight-end
40 |       ],
41 |     },
42 |   },
43 | };
44 | ```
45 | 
46 | The docs version dropdown appears in your navbar:
47 | 
48 | ![Docs Version Dropdown](./img/docsVersionDropdown.png)
49 | 
50 | ## Update an existing version
51 | 
52 | It is possible to edit versioned docs in their respective folder:
53 | 
54 | - `versioned_docs/version-1.0/hello.md` updates `http://localhost:3000/docs/hello`
55 | - `docs/hello.md` updates `http://localhost:3000/docs/next/hello`
56 | 


--------------------------------------------------------------------------------
/docs/my-website/src/pages/tutorial-extras/translate-your-site.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | sidebar_position: 2
 3 | ---
 4 | 
 5 | # Translate your site
 6 | 
 7 | Let's translate `docs/intro.md` to French.
 8 | 
 9 | ## Configure i18n
10 | 
11 | Modify `docusaurus.config.js` to add support for the `fr` locale:
12 | 
13 | ```js title="docusaurus.config.js"
14 | module.exports = {
15 |   i18n: {
16 |     defaultLocale: 'en',
17 |     locales: ['en', 'fr'],
18 |   },
19 | };
20 | ```
21 | 
22 | ## Translate a doc
23 | 
24 | Copy the `docs/intro.md` file to the `i18n/fr` folder:
25 | 
26 | ```bash
27 | mkdir -p i18n/fr/docusaurus-plugin-content-docs/current/
28 | 
29 | cp docs/intro.md i18n/fr/docusaurus-plugin-content-docs/current/intro.md
30 | ```
31 | 
32 | Translate `i18n/fr/docusaurus-plugin-content-docs/current/intro.md` in French.
33 | 
34 | ## Start your localized site
35 | 
36 | Start your site on the French locale:
37 | 
38 | ```bash
39 | npm run start -- --locale fr
40 | ```
41 | 
42 | Your localized site is accessible at [http://localhost:3000/fr/](http://localhost:3000/fr/) and the `Getting Started` page is translated.
43 | 
44 | :::caution
45 | 
46 | In development, you can only use one locale at a same time.
47 | 
48 | :::
49 | 
50 | ## Add a Locale Dropdown
51 | 
52 | To navigate seamlessly across languages, add a locale dropdown.
53 | 
54 | Modify the `docusaurus.config.js` file:
55 | 
56 | ```js title="docusaurus.config.js"
57 | module.exports = {
58 |   themeConfig: {
59 |     navbar: {
60 |       items: [
61 |         // highlight-start
62 |         {
63 |           type: 'localeDropdown',
64 |         },
65 |         // highlight-end
66 |       ],
67 |     },
68 |   },
69 | };
70 | ```
71 | 
72 | The locale dropdown now appears in your navbar:
73 | 
74 | ![Locale Dropdown](./img/localeDropdown.png)
75 | 
76 | ## Build your localized site
77 | 
78 | Build your site for a specific locale:
79 | 
80 | ```bash
81 | npm run build -- --locale fr
82 | ```
83 | 
84 | Or build your site to include all the locales at once:
85 | 
86 | ```bash
87 | npm run build
88 | ```
89 | 


--------------------------------------------------------------------------------
/docs/my-website/static/.nojekyll:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/static/.nojekyll


--------------------------------------------------------------------------------
/docs/my-website/static/img/docusaurus-social-card.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/static/img/docusaurus-social-card.png


--------------------------------------------------------------------------------
/docs/my-website/static/img/docusaurus.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/static/img/docusaurus.png


--------------------------------------------------------------------------------
/docs/my-website/static/img/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/docs/my-website/static/img/favicon.ico


--------------------------------------------------------------------------------
/litellm/.env.template:
--------------------------------------------------------------------------------
 1 | ### KEYS ###
 2 | # HUGGINGFACE_API_KEY="" # Uncomment to save your Hugging Face API key
 3 | # OPENAI_API_KEY="" # Uncomment to save your OpenAI API Key
 4 | # TOGETHERAI_API_KEY="" # Uncomment to save your TogetherAI API key
 5 | # NLP_CLOUD_API_KEY="" # Uncomment to save your NLP Cloud API key
 6 | # ANTHROPIC_API_KEY="" # Uncomment to save your Anthropic API key
 7 | 
 8 | ### MODEL CUSTOM PROMPT TEMPLATE ###
 9 | # MODEL_SYSTEM_MESSAGE_START_TOKEN = "<|prompter|>" # This does not need to be a token, can be any string
10 | # MODEL_SYSTEM_MESSAGE_END_TOKEN = "<|endoftext|>" # This does not need to be a token, can be any string
11 | 
12 | # MODEL_USER_MESSAGE_START_TOKEN = "<|prompter|>" # This does not need to be a token, can be any string
13 | # MODEL_USER_MESSAGE_END_TOKEN = "<|endoftext|>" # Applies only to user messages. Can be any string.
14 | 
15 | # MODEL_ASSISTANT_MESSAGE_START_TOKEN = "<|prompter|>" # Applies only to assistant messages. Can be any string.
16 | # MODEL_ASSISTANT_MESSAGE_END_TOKEN = "<|endoftext|>" # Applies only to system messages. Can be any string.
17 | 
18 | # MODEL_PRE_PROMPT = "You are a good bot" # Applied at the start of the prompt
19 | # MODEL_POST_PROMPT = "Now answer as best as you can" # Applied at the end of the prompt


--------------------------------------------------------------------------------
/litellm/_version.py:
--------------------------------------------------------------------------------
1 | import importlib_metadata
2 | 
3 | try:
4 |     version = importlib_metadata.version("litellm")
5 | except:
6 |     pass
7 | 


--------------------------------------------------------------------------------
/litellm/config.json:
--------------------------------------------------------------------------------
 1 | 
 2 | {
 3 |     "gpt-4": {
 4 |       "prompt": "hello i am a coding assistant",
 5 |       "temperature": 0.7,
 6 |       "max_tokens": 10
 7 |     },
 8 |     "claude-instant-1.2": {
 9 |       "prompt": "hello i am a coding assistant",
10 |       "temperature": 0.7,
11 |       "max_tokens": 10
12 |     }
13 | }
14 |   


--------------------------------------------------------------------------------
/litellm/cost.json:
--------------------------------------------------------------------------------
1 | {
2 |     "gpt-3.5-turbo-0613": 0.00015000000000000001,
3 |     "claude-2": 0.00016454,
4 |     "gpt-4-0613": 0.015408
5 | }


--------------------------------------------------------------------------------
/litellm/gpt_cache.py:
--------------------------------------------------------------------------------
 1 | ###### LiteLLM Integration with GPT Cache #########
 2 | import gptcache
 3 | 
 4 | # openai.ChatCompletion._llm_handler = litellm.completion
 5 | from gptcache.adapter import openai
 6 | import litellm
 7 | 
 8 | 
 9 | class LiteLLMChatCompletion(gptcache.adapter.openai.ChatCompletion):
10 |     @classmethod
11 |     def _llm_handler(cls, *llm_args, **llm_kwargs):
12 |         return litellm.completion(*llm_args, **llm_kwargs)
13 | 
14 | 
15 | completion = LiteLLMChatCompletion.create
16 | ###### End of LiteLLM Integration with GPT Cache #########
17 | 
18 | 
19 | # ####### Example usage ###############
20 | # from gptcache import cache
21 | # completion = LiteLLMChatCompletion.create
22 | # # set API keys in .env / os.environ
23 | # cache.init()
24 | # cache.set_openai_key()
25 | # result = completion(model="claude-2", messages=[{"role": "user", "content": "cto of litellm"}])
26 | # print(result)
27 | 


--------------------------------------------------------------------------------
/litellm/integrations/__init__.py:
--------------------------------------------------------------------------------
1 | from . import *
2 | 


--------------------------------------------------------------------------------
/litellm/integrations/custom_logger.py:
--------------------------------------------------------------------------------
 1 | #### What this does ####
 2 | #    On success, logs events to Promptlayer
 3 | import dotenv, os
 4 | import requests
 5 | import requests
 6 | 
 7 | dotenv.load_dotenv()  # Loading env variables using dotenv
 8 | import traceback
 9 | 
10 | 
11 | class CustomLogger:
12 |     # Class variables or attributes
13 |     def __init__(self):
14 |         pass
15 |     
16 |     def log_input_event(self, model, messages, kwargs, print_verbose, callback_func):
17 |         try: 
18 |             print_verbose(
19 |                     f"Custom Logger - Enters logging function for model {kwargs}"
20 |                 )
21 |             kwargs["model"] = model
22 |             kwargs["messages"] = messages
23 |             kwargs["log_event_type"] = "pre_api_call"
24 |             callback_func(
25 |                 kwargs,
26 |             )
27 |             print_verbose(
28 |                 f"Custom Logger - model call details: {kwargs}"
29 |             )
30 |         except: 
31 |             traceback.print_exc()
32 |             print_verbose(f"Custom Logger Error - {traceback.format_exc()}")
33 | 
34 |     def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose, callback_func):
35 |         # Method definition
36 |         try:
37 |             print_verbose(
38 |                 f"Custom Logger - Enters logging function for model {kwargs}"
39 |             )
40 |             kwargs["log_event_type"] = "post_api_call"
41 |             callback_func(
42 |                 kwargs, # kwargs to func
43 |                 response_obj,
44 |                 start_time,
45 |                 end_time,
46 |             )
47 |             print_verbose(
48 |                 f"Custom Logger - final response object: {response_obj}"
49 |             )
50 |         except:
51 |             # traceback.print_exc()
52 |             print_verbose(f"Custom Logger Error - {traceback.format_exc()}")
53 |             pass
54 | 


--------------------------------------------------------------------------------
/litellm/integrations/traceloop.py:
--------------------------------------------------------------------------------
1 | import sys
2 | 
3 | 
4 | class TraceloopLogger:
5 |     def __init__(self):
6 |         from traceloop.tracing import Tracer
7 | 
8 |         self.tracer = Tracer.init(app_name=sys.argv[0], disable_batch=True)
9 | 


--------------------------------------------------------------------------------
/litellm/llms/__init__.py:
--------------------------------------------------------------------------------
1 | from . import *
2 | 


--------------------------------------------------------------------------------
/litellm/llms/base.py:
--------------------------------------------------------------------------------
 1 | ## This is a template base class to be used for adding new LLM providers via API calls
 2 | import litellm 
 3 | import requests, certifi, ssl
 4 | 
 5 | class BaseLLM:
 6 |     def create_client_session(self):
 7 |         if litellm.client_session: 
 8 |             session = litellm.client_session
 9 |         else: 
10 |             session = requests.Session()
11 |         
12 |         return session
13 |         
14 |     def validate_environment(self):  # set up the environment required to run the model
15 |         pass
16 | 
17 |     def completion(
18 |         self,
19 |     ):  # logic for parsing in - calling - parsing out model completion calls
20 |         pass
21 | 
22 |     def embedding(
23 |         self,
24 |     ):  # logic for parsing in - calling - parsing out model embedding calls
25 |         pass
26 | 


--------------------------------------------------------------------------------
/litellm/proxy/.gitignore:
--------------------------------------------------------------------------------
1 | .env
2 | secrets.toml


--------------------------------------------------------------------------------
/litellm/proxy/README.md:
--------------------------------------------------------------------------------
 1 | # litellm-proxy
 2 | 
 3 | A local, fast, and lightweight **OpenAI-compatible server** to call 100+ LLM APIs.
 4 | 
 5 | ## usage 
 6 | 
 7 | ```shell 
 8 | $ pip install litellm
 9 | ```
10 | ```shell
11 | $ litellm --model ollama/codellama 
12 | 
13 | #INFO: Ollama running on http://0.0.0.0:8000
14 | ```
15 | 
16 | ## replace openai base
17 | ```python 
18 | import openai 
19 | 
20 | openai.api_base = "http://0.0.0.0:8000"
21 | 
22 | print(openai.ChatCompletion.create(model="test", messages=[{"role":"user", "content":"Hey!"}]))
23 | ``` 
24 | 
25 | [**See how to call Huggingface,Bedrock,TogetherAI,Anthropic, etc.**](https://docs.litellm.ai/docs/proxy_server)
26 | 
27 | ## configure proxy
28 | 
29 | To save API Keys, change model prompt, etc. you'll need to create a local instance of it:
30 | ```shell
31 | $ litellm --create-proxy
32 | ```
33 | This will create a local project called `litellm-proxy` in your current directory, that has: 
34 | * **proxy_cli.py**: Runs the proxy
35 | * **proxy_server.py**: Contains the API calling logic
36 |     - `/chat/completions`: receives `openai.ChatCompletion.create` call.
37 |     - `/completions`: receives `openai.Completion.create` call.
38 |     - `/models`: receives `openai.Model.list()` call
39 | * **secrets.toml**: Stores your api keys, model configs, etc.
40 | 
41 | Run it by doing:
42 | ```shell
43 | $ cd litellm-proxy
44 | ```
45 | ```shell
46 | $ python proxy_cli.py --model ollama/llama # replace with your model name
47 | ```


--------------------------------------------------------------------------------
/litellm/proxy/__init__.py:
--------------------------------------------------------------------------------
1 | from . import *


--------------------------------------------------------------------------------
/litellm/proxy/cost.log:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/litellm/proxy/cost.log


--------------------------------------------------------------------------------
/litellm/proxy/costs.json:
--------------------------------------------------------------------------------
1 | {
2 |   "Oct-12-2023": {
3 |     "claude-2": {
4 |       "cost": 0.02365918,
5 |       "num_requests": 1
6 |     }
7 |   }
8 | }


--------------------------------------------------------------------------------
/litellm/proxy/start.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | python3 proxy_cli.py --config -f ../../secrets_template.toml
3 | python3 proxy_cli.py


--------------------------------------------------------------------------------
/litellm/tests/api_log.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "20231020092117501221": {
 3 |     "pre_api_call": {
 4 |       "model": "chatgpt-v-2",
 5 |       "messages": [
 6 |         {
 7 |           "role": "user",
 8 |           "content": "Hey!"
 9 |         }
10 |       ],
11 |       "optional_params": {},
12 |       "litellm_params": {
13 |         "return_async": false,
14 |         "api_key": null,
15 |         "force_timeout": 600,
16 |         "logger_fn": null,
17 |         "verbose": false,
18 |         "custom_llm_provider": "azure",
19 |         "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/",
20 |         "litellm_call_id": "853a40c6-2b0a-41f0-9750-4974d079a6e7",
21 |         "model_alias_map": {},
22 |         "completion_call_id": null,
23 |         "metadata": null,
24 |         "stream_response": {}
25 |       },
26 |       "input": [
27 |         {
28 |           "role": "user",
29 |           "content": "Hey!"
30 |         }
31 |       ],
32 |       "api_key": "6314c6dc63f448c9873844297f408c74",
33 |       "additional_args": {
34 |         "headers": null,
35 |         "api_version": "2023-05-15",
36 |         "api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/"
37 |       },
38 |       "log_event_type": "pre_api_call"
39 |     }
40 |   }
41 | }


--------------------------------------------------------------------------------
/litellm/tests/data_map.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BerriAI/openai-proxy/34c75b8c0c8c200a0c800c19e3ece5ac73b06d18/litellm/tests/data_map.txt


--------------------------------------------------------------------------------
/litellm/tests/litellm_uuid.txt:
--------------------------------------------------------------------------------
1 | 80888ede-4881-4876-ab3f-765d47282e66


--------------------------------------------------------------------------------
/litellm/tests/model_cost.json:
--------------------------------------------------------------------------------
1 | {
2 |     "gpt-3.5-turbo-0613": 7.7e-05
3 | }


--------------------------------------------------------------------------------
/litellm/tests/test_api_key_param.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests the ability to set api key's via the params instead of as environment variables
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | 
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | import litellm
11 | from litellm import embedding, completion
12 | 
13 | litellm.set_verbose = False
14 | 
15 | 
16 | def logger_fn(model_call_object: dict):
17 |     print(f"model call details: {model_call_object}")
18 | 
19 | 
20 | user_message = "Hello, how are you?"
21 | messages = [{"content": user_message, "role": "user"}]
22 | 
23 | ## Test 1: Setting key dynamically
24 | temp_key = os.environ.get("ANTHROPIC_API_KEY", "")
25 | os.environ["ANTHROPIC_API_KEY"] = "bad-key"
26 | # test on openai completion call
27 | try:
28 |     response = completion(
29 |         model="claude-instant-1",
30 |         messages=messages,
31 |         logger_fn=logger_fn,
32 |         api_key=temp_key,
33 |     )
34 |     print(f"response: {response}")
35 | except:
36 |     print(f"error occurred: {traceback.format_exc()}")
37 |     pass
38 | os.environ["ANTHROPIC_API_KEY"] = temp_key
39 | 
40 | 
41 | ## Test 2: Setting key via __init__ params
42 | litellm.anthropic_key = os.environ.get("ANTHROPIC_API_KEY", "")
43 | os.environ.pop("ANTHROPIC_API_KEY")
44 | # test on openai completion call
45 | try:
46 |     response = completion(
47 |         model="claude-instant-1", messages=messages, logger_fn=logger_fn
48 |     )
49 |     print(f"response: {response}")
50 | except:
51 |     print(f"error occurred: {traceback.format_exc()}")
52 |     pass
53 | os.environ["ANTHROPIC_API_KEY"] = temp_key
54 | 


--------------------------------------------------------------------------------
/litellm/tests/test_async_fn.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests the the acompletion function #
 3 | 
 4 | import sys, os
 5 | import pytest
 6 | import traceback
 7 | import asyncio
 8 | 
 9 | sys.path.insert(
10 |     0, os.path.abspath("../..")
11 | )  # Adds the parent directory to the system path
12 | from litellm import acompletion, acreate
13 | 
14 | @pytest.mark.asyncio
15 | async def test_get_response():
16 |     user_message = "Hello, how are you?"
17 |     messages = [{"content": user_message, "role": "user"}]
18 |     try:
19 |         response = await acompletion(model="gpt-3.5-turbo", messages=messages)
20 |     except Exception as e:
21 |         pass
22 | 
23 | response = asyncio.run(test_get_response())
24 | # print(response)
25 | 
26 | @pytest.mark.asyncio
27 | async def test_get_response_streaming():
28 |     user_message = "Hello, how are you?"
29 |     messages = [{"content": user_message, "role": "user"}]
30 |     try:
31 |         response = await acompletion(model="gpt-3.5-turbo", messages=messages, stream=True)
32 |         print(type(response))
33 | 
34 |         import inspect
35 | 
36 |         is_async_generator = inspect.isasyncgen(response)
37 |         print(is_async_generator)
38 | 
39 |         output = ""
40 |         async for chunk in response:
41 |             token = chunk["choices"][0]["delta"].get("content", "")
42 |             output += token
43 |             print(output)
44 | 
45 |         assert output is not None, "output cannot be None."
46 |         assert isinstance(output, str), "output needs to be of type str"
47 |         assert len(output) > 0, "Length of output needs to be greater than 0."
48 | 
49 |     except Exception as e:
50 |         pass
51 |     return response
52 | 
53 | # response = asyncio.run(test_get_response_streaming())
54 | # print(response)
55 | 
56 | 
57 | 


--------------------------------------------------------------------------------
/litellm/tests/test_get_model_cost_map.py:
--------------------------------------------------------------------------------
 1 | import sys, os
 2 | import traceback
 3 | 
 4 | sys.path.insert(
 5 |     0, os.path.abspath("../..")
 6 | )  # Adds the parent directory to the system path
 7 | import time
 8 | from litellm import get_max_tokens, model_cost, open_ai_chat_completion_models
 9 | 
10 | print(get_max_tokens("gpt-3.5-turbo"))
11 | 
12 | print(model_cost)
13 | print(open_ai_chat_completion_models)
14 | 


--------------------------------------------------------------------------------
/litellm/tests/test_get_model_file.py:
--------------------------------------------------------------------------------
 1 | import os, sys, traceback
 2 | 
 3 | sys.path.insert(
 4 |     0, os.path.abspath("../..")
 5 | )  # Adds the parent directory to the system path
 6 | import litellm
 7 | import pytest
 8 | 
 9 | try:
10 |     print(litellm.get_model_cost_map(url="fake-url"))
11 | except Exception as e:
12 |     pytest.fail(f"An exception occurred: {e}")


--------------------------------------------------------------------------------
/litellm/tests/test_get_model_list.py:
--------------------------------------------------------------------------------
 1 | import os, sys, traceback
 2 | 
 3 | sys.path.insert(
 4 |     0, os.path.abspath("../..")
 5 | )  # Adds the parent directory to the system path
 6 | import litellm
 7 | from litellm import get_model_list
 8 | 
 9 | print(get_model_list())
10 | print(get_model_list())
11 | # print(litellm.model_list)
12 | 


--------------------------------------------------------------------------------
/litellm/tests/test_helicone_integration.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests if logging to the helicone integration actually works
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | import pytest
 7 | 
 8 | sys.path.insert(
 9 |     0, os.path.abspath("../..")
10 | )  # Adds the parent directory to the system path
11 | import litellm
12 | from litellm import embedding, completion
13 | 
14 | litellm.success_callback = ["helicone"]
15 | 
16 | litellm.set_verbose = True
17 | 
18 | user_message = "Hello, how are you?"
19 | messages = [{"content": user_message, "role": "user"}]
20 | 
21 | 
22 | # openai call
23 | response = completion(
24 |     model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]
25 | )
26 | 
27 | # cohere call
28 | response = completion(
29 |     model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}]
30 | )
31 | 


--------------------------------------------------------------------------------
/litellm/tests/test_hf_prompt_templates.py:
--------------------------------------------------------------------------------
 1 | import sys, os
 2 | import traceback
 3 | from dotenv import load_dotenv
 4 | 
 5 | load_dotenv()
 6 | import os
 7 | 
 8 | sys.path.insert(
 9 |     0, os.path.abspath("../..")
10 | )  # Adds the parent directory to the system path
11 | import pytest
12 | from litellm.llms.prompt_templates.factory import prompt_factory
13 | 
14 | def test_prompt_formatting(): 
15 |     try:
16 |         prompt = prompt_factory(model="mistralai/Mistral-7B-Instruct-v0.1", messages=[{"role": "system", "content": "Be a good bot"}, {"role": "user", "content": "Hello world"}])
17 |         assert prompt == "[INST] Be a good bot [/INST] [INST] Hello world [/INST]"
18 |     except Exception as e: 
19 |         pytest.fail(f"An exception occurred: {str(e)}")
20 | # def logger_fn(user_model_dict):
21 | #     return
22 | #     print(f"user_model_dict: {user_model_dict}")
23 | 
24 | # messages=[{"role": "user", "content": "Write me a function to print hello world"}]
25 | 
26 | # # test if the first-party prompt templates work 
27 | # def test_huggingface_supported_models():
28 | #     model = "huggingface/WizardLM/WizardCoder-Python-34B-V1.0"
29 | #     response = completion(model=model, messages=messages, max_tokens=256, api_base="https://ji16r2iys9a8rjk2.us-east-1.aws.endpoints.huggingface.cloud", logger_fn=logger_fn)
30 | #     print(response['choices'][0]['message']['content'])
31 | #     return response
32 | 
33 | # test_huggingface_supported_models()
34 | 
35 | # # test if a custom prompt template works 
36 | # litellm.register_prompt_template(
37 | # 	model="togethercomputer/LLaMA-2-7B-32K",
38 | # 	roles={"system":"", "assistant":"Assistant:", "user":"User:"},
39 | # 	pre_message_sep= "\n",
40 | # 	post_message_sep= "\n"
41 | # )
42 | # def test_huggingface_custom_model():
43 | #     model = "huggingface/togethercomputer/LLaMA-2-7B-32K"
44 | #     response = completion(model=model, messages=messages, api_base="https://ecd4sb5n09bo4ei2.us-east-1.aws.endpoints.huggingface.cloud", logger_fn=logger_fn)
45 | #     print(response['choices'][0]['message']['content'])
46 | #     return response
47 | 
48 | # test_huggingface_custom_model()


--------------------------------------------------------------------------------
/litellm/tests/test_langsmith.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | import os
 3 | import io
 4 | 
 5 | sys.path.insert(0, os.path.abspath('../..'))
 6 | 
 7 | from litellm import completion
 8 | import litellm
 9 | 
10 | litellm.success_callback = ["langsmith"]
11 | # litellm.set_verbose = True
12 | import time
13 | 
14 | 
15 | def test_langsmith_logging():
16 |     try:
17 |         response = completion(model="claude-instant-1.2",
18 |                               messages=[{
19 |                                   "role": "user",
20 |                                   "content": "what llm are u"
21 |                               }],
22 |                               max_tokens=10,
23 |                               temperature=0.2
24 |                               )
25 |         print(response)
26 |     except Exception as e:
27 |         print(e)
28 | 
29 | test_langsmith_logging()
30 | 
31 | 
32 | def test_langsmith_logging_with_metadata():
33 |     try:
34 |         response = completion(model="gpt-3.5-turbo",
35 |                               messages=[{
36 |                                   "role": "user",
37 |                                   "content": "what llm are u"
38 |                               }],
39 |                               max_tokens=10,
40 |                               temperature=0.2,
41 |                               metadata={
42 |                                   "run_name": "litellmRUN",
43 |                                   "project_name": "litellm-completion",
44 |                               }
45 |                               )
46 |         print(response)
47 |     except Exception as e:
48 |         print(e)
49 | 
50 | test_langsmith_logging_with_metadata()
51 | 


--------------------------------------------------------------------------------
/litellm/tests/test_litellm_max_budget.py:
--------------------------------------------------------------------------------
 1 | # #### What this tests ####
 2 | # #    This tests calling litellm.max_budget by making back-to-back gpt-4 calls
 3 | # # commenting out this test for circle ci, as it causes other tests to fail, since litellm.max_budget would impact other litellm imports
 4 | # import sys, os, json
 5 | # import traceback
 6 | # import pytest 
 7 | 
 8 | # sys.path.insert(
 9 | #     0, os.path.abspath("../..")
10 | # )  # Adds the parent directory to the system path
11 | # import litellm 
12 | # # litellm.set_verbose = True
13 | # from litellm import completion, BudgetExceededError
14 | 
15 | # def test_max_budget():
16 | #     try:
17 | #         litellm.max_budget = 0.001 # sets a max budget of $0.001
18 | 
19 | #         messages = [{"role": "user", "content": "Hey, how's it going"}]
20 | #         response = completion(model="gpt-4", messages=messages, stream=True)
21 | #         for chunk in response: 
22 | #             continue
23 | #         print(litellm._current_cost)
24 | #         completion(model="gpt-4", messages=messages, stream=True)
25 | #         litellm.max_budget = float('inf')
26 | #     except BudgetExceededError as e: 
27 | #         pass
28 | #     except Exception as e:
29 | #         pytest.fail(f"An error occured: {str(e)}")
30 | 
31 | 
32 | 


--------------------------------------------------------------------------------
/litellm/tests/test_load_test_model.py:
--------------------------------------------------------------------------------
 1 | import sys, os
 2 | import traceback
 3 | 
 4 | sys.path.insert(
 5 |     0, os.path.abspath("../..")
 6 | )  # Adds the parent directory to the system path
 7 | import litellm
 8 | from litellm import load_test_model, testing_batch_completion
 9 | 
10 | # ## Load Test Model
11 | # model="gpt-3.5-turbo"
12 | # result = load_test_model(model=model, num_calls=5)
13 | # print(result)
14 | # print(len(result["results"]))
15 | 
16 | # ## Duration Test Model
17 | # model="gpt-3.5-turbo"
18 | # result = load_test_model(model=model, num_calls=5, duration=15, interval=15) # duration test the model for 2 minutes, sending 5 calls every 15s
19 | # print(result)
20 | 
21 | ## Quality Test across Model
22 | models = [
23 |     "gpt-3.5-turbo",
24 |     "gpt-3.5-turbo-16k",
25 |     "gpt-4",
26 |     "claude-instant-1",
27 |     {
28 |         "model": "replicate/llama-2-70b-chat:58d078176e02c219e11eb4da5a02a7830a283b14cf8f94537af893ccff5ee781",
29 |         "custom_llm_provider": "replicate",
30 |     },
31 | ]
32 | messages = [
33 |     [{"role": "user", "content": "What is your name?"}],
34 |     [{"role": "user", "content": "Hey, how's it going?"}],
35 | ]
36 | result = testing_batch_completion(models=models, messages=messages)
37 | print(result)
38 | 


--------------------------------------------------------------------------------
/litellm/tests/test_longer_context_fallback.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests context fallback dict
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | import pytest
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | import litellm
11 | from litellm import longer_context_model_fallback_dict
12 | 
13 | print(longer_context_model_fallback_dict)


--------------------------------------------------------------------------------
/litellm/tests/test_mock_request.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests mock request calls to litellm 
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | 
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | import litellm
11 | 
12 | def test_mock_request():
13 |     try:
14 |         model = "gpt-3.5-turbo"
15 |         messages = [{"role": "user", "content": "Hey, I'm a mock request"}]
16 |         response = litellm.mock_completion(model=model, messages=messages, stream=False)
17 |         print(response)
18 |         print(type(response))
19 |     except:
20 |         traceback.print_exc()
21 | 
22 | # test_mock_request()
23 | def test_streaming_mock_request():
24 |     try: 
25 |         model = "gpt-3.5-turbo"
26 |         messages = [{"role": "user", "content": "Hey, I'm a mock request"}]
27 |         response = litellm.mock_completion(model=model, messages=messages, stream=True)
28 |         complete_response = "" 
29 |         for chunk in response: 
30 |             complete_response += chunk["choices"][0]["delta"]["content"]
31 |         if complete_response == "": 
32 |             raise Exception("Empty response received")
33 |     except:
34 |         traceback.print_exc()
35 | 
36 | test_streaming_mock_request()


--------------------------------------------------------------------------------
/litellm/tests/test_model_alias_map.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests the model alias mapping - if user passes in an alias, and has set an alias, set it to the actual value
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | 
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | import litellm
11 | from litellm import embedding, completion
12 | 
13 | litellm.set_verbose = True
14 | 
15 | model_alias_map = {
16 |     "llama2": "replicate/llama-2-70b-chat:2796ee9483c3fd7aa2e171d38f4ca12251a30609463dcfd4cd76703f22e96cdf"
17 | }
18 | 
19 | litellm.model_alias_map = model_alias_map
20 | 
21 | try:
22 |     completion(
23 |         "llama2",
24 |         messages=[{"role": "user", "content": "Hey, how's it going?"}],
25 |         top_p=0.1,
26 |         temperature=0.01,
27 |         num_beams=4,
28 |         max_tokens=60,
29 |     )
30 | except Exception as e:
31 |     print(e)
32 | 


--------------------------------------------------------------------------------
/litellm/tests/test_model_fallback.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests error handling + logging (esp. for sentry breadcrumbs)
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | 
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | import litellm
11 | from litellm import embedding, completion
12 | 
13 | litellm.success_callback = ["posthog"]
14 | litellm.failure_callback = ["sentry", "posthog"]
15 | 
16 | litellm.set_verbose = True
17 | 
18 | model_fallback_list = ["claude-instant-1", "gpt-3.5-turbo", "chatgpt-test"]
19 | 
20 | user_message = "Hello, how are you?"
21 | messages = [{"content": user_message, "role": "user"}]
22 | 
23 | for model in model_fallback_list:
24 |     try:
25 |         response = embedding(model="text-embedding-ada-002", input=[user_message])
26 |         response = completion(model=model, messages=messages)
27 |     except Exception as e:
28 |         print(f"error occurred: {traceback.format_exc()}")
29 | 


--------------------------------------------------------------------------------
/litellm/tests/test_model_response_typing/server.py:
--------------------------------------------------------------------------------
 1 | # #### What this tests ####
 2 | # #    This tests if the litellm model response type is returnable in a flask app
 3 | 
 4 | # import sys, os
 5 | # import traceback
 6 | # from flask import Flask, request, jsonify, abort, Response
 7 | # sys.path.insert(0, os.path.abspath('../../..'))  # Adds the parent directory to the system path
 8 | 
 9 | # import litellm
10 | # from litellm import completion
11 | 
12 | # litellm.set_verbose = False
13 | 
14 | # app = Flask(__name__)
15 | 
16 | # @app.route('/')
17 | # def hello():
18 | #     data = request.json
19 | #     return completion(**data)
20 | 
21 | # if __name__ == '__main__':
22 | #     from waitress import serve
23 | #     serve(app, host='localhost', port=8080, threads=10)
24 | 


--------------------------------------------------------------------------------
/litellm/tests/test_model_response_typing/test.py:
--------------------------------------------------------------------------------
 1 | # import requests, json
 2 | 
 3 | # BASE_URL = 'http://localhost:8080'
 4 | 
 5 | # def test_hello_route():
 6 | #     data = {"model": "claude-instant-1", "messages": [{"role": "user", "content": "hey, how's it going?"}]}
 7 | #     headers = {'Content-Type': 'application/json'}
 8 | #     response = requests.get(BASE_URL, headers=headers, data=json.dumps(data))
 9 | #     print(response.text)
10 | #     assert response.status_code == 200
11 | #     print("Hello route test passed!")
12 | 
13 | # if __name__ == '__main__':
14 | #     test_hello_route()
15 | 


--------------------------------------------------------------------------------
/litellm/tests/test_multiple_deployments.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests error handling + logging (esp. for sentry breadcrumbs)
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | 
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | import pytest
11 | from litellm import completion
12 | 
13 | messages=[{"role": "user", "content": "Hey, how's it going?"}]
14 | 
15 | ## All your mistral deployments ##
16 | model_list = [{
17 | 	"model_name": "mistral-7b-instruct",
18 | 	"litellm_params": { # params for litellm completion/embedding call 
19 |         "model": "replicate/mistralai/mistral-7b-instruct-v0.1:83b6a56e7c828e667f21fd596c338fd4f0039b46bcfa18d973e8e70e455fda70", 
20 |         "api_key": os.getenv("REPLICATE_API_KEY"),
21 |     }
22 | }, {
23 | 	"model_name": "mistral-7b-instruct",
24 | 	"litellm_params": { # params for litellm completion/embedding call 
25 |         "model": "together_ai/mistralai/Mistral-7B-Instruct-v0.1", 
26 |         "api_key": os.getenv("TOGETHERAI_API_KEY"),
27 |     }
28 | }, {
29 | 	"model_name": "mistral-7b-instruct",
30 | 	"litellm_params": { # params for litellm completion/embedding call 
31 |         "model": "mistral-7b-instruct", 
32 |         "api_base": "https://api.perplexity.ai",
33 |         "api_key": os.getenv("PERPLEXITYAI_API_KEY")
34 |     }
35 | }, {
36 | 	"model_name": "mistral-7b-instruct",
37 | 	"litellm_params": {
38 | 		"model": "deepinfra/mistralai/Mistral-7B-Instruct-v0.1",
39 | 		"api_key": os.getenv("DEEPINFRA_API_KEY")
40 | 	}
41 | }]
42 | 
43 | def test_multiple_deployments():
44 |     try: 
45 |         ## LiteLLM completion call ## returns first response 
46 |         response = completion(model="mistral-7b-instruct", messages=messages, model_list=model_list)
47 |         print(f"response: {response}")
48 |     except Exception as e:
49 |         traceback.print_exc()
50 |         pytest.fail(f"An exception occurred: {e}")
51 | 
52 | test_multiple_deployments()


--------------------------------------------------------------------------------
/litellm/tests/test_no_client.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests error logging (with custom user functions) for the `completion` + `embedding` endpoints without callbacks (i.e. slack, posthog, etc. not set)
 3 | #    Requirements: Remove any env keys you have related to slack/posthog/etc. + anthropic api key (cause an exception)
 4 | 
 5 | import sys, os
 6 | import traceback
 7 | 
 8 | sys.path.insert(
 9 |     0, os.path.abspath("../..")
10 | )  # Adds the parent directory to the system path
11 | import litellm
12 | from litellm import embedding, completion
13 | 
14 | litellm.set_verbose = True
15 | 
16 | model_fallback_list = ["claude-instant-1", "gpt-3.5-turbo", "chatgpt-test"]
17 | 
18 | user_message = "Hello, how are you?"
19 | messages = [{"content": user_message, "role": "user"}]
20 | 
21 | for model in model_fallback_list:
22 |     try:
23 |         response = embedding(model="text-embedding-ada-002", input=[user_message])
24 |         response = completion(model=model, messages=messages)
25 |     except Exception as e:
26 |         print(f"error occurred: {traceback.format_exc()}")
27 | 


--------------------------------------------------------------------------------
/litellm/tests/test_prompt_factory.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests if prompts are being correctly formatted
 3 | import sys
 4 | import os
 5 | import io
 6 | 
 7 | sys.path.insert(0, os.path.abspath('../..'))
 8 | 
 9 | # from litellm.llms.prompt_templates.factory import prompt_factory
10 | from litellm import completion
11 | 
12 | def codellama_prompt_format():
13 |     model = "huggingface/codellama/CodeLlama-7b-Instruct-hf"
14 |     messages = [{"role": "system", "content": "You are a good bot"}, {"role": "user", "content": "Hey, how's it going?"}]
15 |     expected_response = """[INST] <>
16 | You are a good bot
17 | <>
18 |  [/INST]
19 | [INST] Hey, how's it going? [/INST]"""
20 |     response = completion(model=model, messages=messages)
21 |     print(response)
22 | 
23 | # codellama_prompt_format()


--------------------------------------------------------------------------------
/litellm/tests/test_proxy_api.py:
--------------------------------------------------------------------------------
 1 | # import sys, os
 2 | # import traceback
 3 | # sys.path.insert(0, os.path.abspath('../..'))  # Adds the parent directory to the system path
 4 | # import litellm
 5 | # from litellm import embedding, completion
 6 | 
 7 | # litellm.api_base = "https://oai.hconeai.com/v1"
 8 | # litellm.headers = {"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}"}
 9 | 
10 | # response = litellm.completion(
11 | #     model="gpt-3.5-turbo",
12 | #     messages=[{"role": "user", "content": "how does a court case get to the Supreme Court?"}]
13 | # )
14 | 
15 | # print(response)
16 | 


--------------------------------------------------------------------------------
/litellm/tests/test_register_model.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests calling batch_completions by running 100 messages together
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | import pytest
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | import litellm
11 | 
12 | def test_update_model_cost():
13 |     try: 
14 |         litellm.register_model({
15 |              "gpt-4": {
16 |                 "max_tokens": 8192, 
17 |                 "input_cost_per_token": 0.00002, 
18 |                 "output_cost_per_token": 0.00006, 
19 |                 "litellm_provider": "openai", 
20 |                 "mode": "chat"
21 |             },
22 |         })
23 |         assert litellm.model_cost["gpt-4"]["input_cost_per_token"] == 0.00002
24 |     except Exception as e:
25 |         pytest.fail(f"An error occurred: {e}")
26 | 
27 | test_update_model_cost()


--------------------------------------------------------------------------------
/litellm/tests/test_router.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests calling batch_completions by running 100 messages together
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | import pytest
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | from litellm import Router
11 | from concurrent.futures import ThreadPoolExecutor
12 | from dotenv import load_dotenv
13 | 
14 | load_dotenv()
15 | 
16 | model_list = [{ # list of model deployments 
17 | 	"model_name": "gpt-3.5-turbo", # openai model name 
18 | 	"litellm_params": { # params for litellm completion/embedding call 
19 | 		"model": "azure/chatgpt-v-2", 
20 | 		"api_key": os.getenv("AZURE_API_KEY"),
21 | 		"api_version": os.getenv("AZURE_API_VERSION"),
22 | 		"api_base": os.getenv("AZURE_API_BASE")
23 | 	},
24 | 	"tpm": 240000,
25 | 	"rpm": 1800
26 | }, {
27 |     "model_name": "gpt-3.5-turbo", # openai model name 
28 | 	"litellm_params": { # params for litellm completion/embedding call 
29 | 		"model": "azure/chatgpt-functioncalling", 
30 | 		"api_key": os.getenv("AZURE_API_KEY"),
31 | 		"api_version": os.getenv("AZURE_API_VERSION"),
32 | 		"api_base": os.getenv("AZURE_API_BASE")
33 | 	},
34 | 	"tpm": 240000,
35 | 	"rpm": 1800
36 | }, {
37 |     "model_name": "gpt-3.5-turbo", # openai model name 
38 | 	"litellm_params": { # params for litellm completion/embedding call 
39 | 		"model": "gpt-3.5-turbo", 
40 | 		"api_key": os.getenv("OPENAI_API_KEY"),
41 | 	},
42 | 	"tpm": 1000000,
43 | 	"rpm": 9000
44 | }]
45 | 
46 | router = Router(model_list=model_list, redis_host=os.getenv("REDIS_HOST"), redis_password=os.getenv("REDIS_PASSWORD"), redis_port=int(os.getenv("REDIS_PORT"))) # type: ignore
47 | 
48 | completions = [] 
49 | with ThreadPoolExecutor(max_workers=100) as executor:
50 | 	kwargs = {
51 | 		"model": "gpt-3.5-turbo",
52 | 		"messages": [{"role": "user", "content": "Hey, how's it going?"}]
53 | 	}
54 | 	for _ in range(20):
55 | 		future = executor.submit(router.completion, **kwargs) # type: ignore
56 | 		completions.append(future)
57 | 
58 | # Retrieve the results from the futures
59 | results = [future.result() for future in completions]
60 | 
61 | print(results)


--------------------------------------------------------------------------------
/litellm/tests/test_stream_chunk_builder.py:
--------------------------------------------------------------------------------
 1 | from litellm import completion, stream_chunk_builder
 2 | import litellm
 3 | import os, dotenv
 4 | dotenv.load_dotenv()
 5 | 
 6 | user_message = "What is the current weather in Boston?"
 7 | messages = [{"content": user_message, "role": "user"}]
 8 | 
 9 | function_schema = {
10 |   "name": "get_weather",
11 |   "description":
12 |   "gets the current weather",
13 |   "parameters": {
14 |     "type": "object",
15 |     "properties": {
16 |       "location": {
17 |         "type": "string",
18 |         "description":
19 |         "The city and state, e.g. San Francisco, CA"
20 |       },
21 |     },
22 |     "required": ["location"]
23 |   },
24 | }
25 | 
26 | def test_stream_chunk_builder():
27 |     litellm.api_key = os.environ["OPENAI_API_KEY"]
28 |     response = completion(
29 |         model="gpt-3.5-turbo",
30 |         messages=messages,
31 |         functions=[function_schema],
32 |         stream=True,
33 |     )
34 | 
35 |     chunks = []
36 | 
37 |     for chunk in response:
38 |         chunks.append(chunk)
39 | 
40 |     try:
41 |         rebuilt_response = stream_chunk_builder(chunks)
42 | 
43 |         # exract the response from the rebuilt response
44 |         rebuilt_response["id"]
45 |         rebuilt_response["object"]
46 |         rebuilt_response["created"]
47 |         rebuilt_response["model"]
48 |         rebuilt_response["choices"]
49 |         rebuilt_response["choices"][0]["index"]
50 |         choices = rebuilt_response["choices"][0]
51 |         message = choices["message"]
52 |         role = message["role"]
53 |         content = message["content"]
54 |         finnish_reason = choices["finish_reason"]
55 |     except:
56 |         raise Exception("stream_chunk_builder failed to rebuild response")
57 | # test_stream_chunk_builder()
58 | 
59 | 


--------------------------------------------------------------------------------
/litellm/tests/test_supabase_integration.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests if logging to the supabase integration actually works
 3 | import sys, os
 4 | import traceback
 5 | import pytest
 6 | 
 7 | sys.path.insert(0, os.path.abspath('../..'))  # Adds the parent directory to the system path
 8 | import litellm
 9 | from litellm import embedding, completion
10 | 
11 | litellm.input_callback = ["supabase"]
12 | litellm.success_callback = ["supabase"]
13 | litellm.failure_callback = ["supabase"]
14 | 
15 | 
16 | litellm.set_verbose = False
17 | 
18 | def test_supabase_logging():
19 |     try:
20 |         response = completion(
21 |             model="gpt-3.5-turbo",
22 |             messages=[{"role": "user", "content": "Hello tell me hi"}],
23 |             user="ishaanRegular",
24 |             max_tokens=10,
25 |         )
26 |         print(response)
27 |     except Exception as e:
28 |         print(e)
29 | 
30 | # test_supabase_logging()
31 | 
32 | def test_acompletion_sync():
33 |     import asyncio
34 |     import time
35 |     async def completion_call():
36 |         try:
37 |             response = await litellm.acompletion(
38 |                 model="gpt-3.5-turbo", 
39 |                 messages=[{"role": "user", "content": "write a poem"}], 
40 |                 max_tokens=10,
41 |                 stream=True,
42 |                 user="ishaanStreamingUser"
43 |             )
44 |             complete_response = ""
45 |             start_time = time.time()
46 |             async for chunk in response:
47 |                 chunk_time = time.time()
48 |                 #print(chunk)
49 |                 complete_response += chunk["choices"][0]["delta"].get("content", "")
50 |                 #print(complete_response)
51 |                 #print(f"time since initial request: {chunk_time - start_time:.5f}")
52 | 
53 |                 if chunk["choices"][0].get("finish_reason", None) != None:
54 |                     print("🤗🤗🤗 DONE")
55 |                     return
56 | 
57 |         except:
58 |             print(f"error occurred: {traceback.format_exc()}")
59 |             pass
60 | 
61 | 
62 |     asyncio.run(completion_call())
63 | # test_acompletion_sync()
64 | 
65 | 
66 | 
67 | 
68 | 


--------------------------------------------------------------------------------
/litellm/tests/test_timeout.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests the timeout decorator
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | 
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | import time
11 | from litellm import timeout
12 | 
13 | 
14 | @timeout(10)
15 | def stop_after_10_s(force_timeout=60):
16 |     print("Stopping after 10 seconds")
17 |     time.sleep(10)
18 |     return
19 | 
20 | 
21 | start_time = time.time()
22 | 
23 | try:
24 |     stop_after_10_s(force_timeout=1)
25 | except Exception as e:
26 |     print(e)
27 |     pass
28 | 
29 | end_time = time.time()
30 | 
31 | print(f"total time: {end_time-start_time}")
32 | 


--------------------------------------------------------------------------------
/litellm/tests/test_validate_environment.py:
--------------------------------------------------------------------------------
 1 | #### What this tests ####
 2 | #    This tests the validate environment function
 3 | 
 4 | import sys, os
 5 | import traceback
 6 | 
 7 | sys.path.insert(
 8 |     0, os.path.abspath("../..")
 9 | )  # Adds the parent directory to the system path
10 | import time
11 | import litellm
12 | 
13 | print(litellm.validate_environment("openai/gpt-3.5-turbo"))


--------------------------------------------------------------------------------
/litellm/tests/test_vertex.py:
--------------------------------------------------------------------------------
 1 | # import sys, os
 2 | # import traceback
 3 | # from dotenv import load_dotenv
 4 | # load_dotenv()
 5 | # import os
 6 | # sys.path.insert(0, os.path.abspath('../..'))  # Adds the parent directory to the system path
 7 | # import pytest
 8 | # import litellm
 9 | # from litellm import embedding, completion
10 | 
11 | 
12 | # litellm.vertex_project = "hardy-device-386718"
13 | # litellm.vertex_location = "us-central1"
14 | # litellm.set_verbose = True
15 | 
16 | # user_message = "what's the weather in SF "
17 | # messages = [{ "content": user_message,"role": "user"}]
18 | # def logger_fn(user_model_dict):
19 | #     print(f"user_model_dict: {user_model_dict}")
20 | 
21 | # # chat-bison
22 | # # response = completion(model="chat-bison", messages=messages, temperature=0.5, top_p=0.1)
23 | # # print(response)
24 | 
25 | # #text-bison
26 | 
27 | # response = completion(model="text-bison", messages=messages)
28 | # print(response)
29 | 
30 | # response = completion(model="text-bison@001", messages=messages, temperature=0.1, logger_fn=logger_fn)
31 | # print(response)
32 | 
33 | # response = completion(model="text-bison", messages=messages, temperature=0.4, top_p=0.1, logger_fn=logger_fn)
34 | # print(response)
35 | 
36 | # response = completion(model="text-bison", messages=messages, temperature=0.8, top_p=0.4, top_k=30, logger_fn=logger_fn)
37 | # print(response)
38 | 
39 | # response = completion(model="text-bison@001", messages=messages, temperature=0.8, top_p=0.4, top_k=30, logger_fn=logger_fn)
40 | # print(response)
41 | 
42 | # # chat_model = ChatModel.from_pretrained("chat-bison@001")
43 | # # parameters = {
44 | # #     "temperature": 0.2,
45 | # #     "max_output_tokens": 256,
46 | # #     "top_p": 0.8,
47 | # #     "top_k": 40
48 | # # }
49 | 
50 | # # chat = chat_model.start_chat()
51 | # # response = chat.send_message("who are u? write a sentence", **parameters)
52 | # # print(f"Response from Model: {response.text}")
53 | 


--------------------------------------------------------------------------------
/litellm/tests/test_wandb_logger.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | import os
 3 | import io
 4 | 
 5 | sys.path.insert(0, os.path.abspath('../..'))
 6 | 
 7 | from litellm import completion
 8 | import litellm
 9 | 
10 | litellm.success_callback = ["wandb"]
11 | 
12 | # litellm.set_verbose = True
13 | import time
14 | 
15 | 
16 | 
17 | def test_wandb_logging():
18 |     try:
19 |         response = completion(model="claude-instant-1.2",
20 |                               messages=[{
21 |                                   "role": "user",
22 |                                   "content": "Hi 👋 - i'm claude"
23 |                               }],
24 |                               max_tokens=10,
25 |                               temperature=0.2
26 |                               )
27 |         # print(response)
28 |     except Exception as e:
29 |         print(e)
30 | 
31 | test_wandb_logging()
32 | 
33 | 
34 | 
35 | 
36 | 
37 | 


--------------------------------------------------------------------------------
/litellm/tests/user_cost.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "1234": {
 3 |         "total_budget": 10,
 4 |         "current_cost": 7.3e-05,
 5 |         "model_cost": {
 6 |             "gpt-3.5-turbo-0613": 7.3e-05
 7 |         }
 8 |     },
 9 |     "12345": {
10 |         "total_budget": 0
11 |     }
12 | }


--------------------------------------------------------------------------------
/openai-proxy/.env.template:
--------------------------------------------------------------------------------
 1 | OPENAI_API_KEY = ""
 2 | 
 3 | AZURE_API_KEY = ""
 4 | AZURE_API_BASE = ""
 5 | AZURE_API_VERSION = ""
 6 | 
 7 | ANTHROPIC_API_KEY = ""
 8 | 
 9 | COHERE_API_KEY = ""
10 | 
11 | 


--------------------------------------------------------------------------------
/openai-proxy/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:3.10
 2 | 
 3 | ENV LITELLM_CONFIG_PATH="/litellm.secrets.toml"
 4 | COPY . /app
 5 | WORKDIR /app
 6 | RUN pip install -r requirements.txt
 7 | 
 8 | EXPOSE $PORT 
 9 | 
10 | CMD exec uvicorn main:app --host 0.0.0.0 --port $PORT


--------------------------------------------------------------------------------
/openai-proxy/README.md:
--------------------------------------------------------------------------------
 1 | # Openai-proxy
 2 | 
 3 | A simple, fast, and lightweight **OpenAI-compatible server** to call 100+ LLM APIs.
 4 | 
 5 | 

 6 |         
 7 |                 
 8 |         
 9 |         
10 |                 
11 |         
12 | 

13 | 
14 | ## Usage 
15 | 
16 | ```shell 
17 | $ git clone https://github.com/BerriAI/litellm.git
18 | ```
19 | ```shell
20 | $ cd ./litellm/openai-proxy
21 | ```
22 | 
23 | ```shell
24 | $ uvicorn main:app --host 0.0.0.0 --port 8000
25 | ```
26 | 
27 | ## Endpoints:
28 | - `/chat/completions` - chat completions endpoint to call 100+ LLMs
29 | - `/models` - available models on server
30 | 
31 | ## Making Requests to Proxy
32 | ### Curl
33 | ```shell
34 | curl http://0.0.0.0:8000/v1/chat/completions \
35 |   -H "Content-Type: application/json" \
36 |   -d '{
37 |      "model": "gpt-3.5-turbo",
38 |      "messages": [{"role": "user", "content": "Say this is a test!"}],
39 |      "temperature": 0.7
40 |    }'
41 | ```
42 | 
43 | ### Replace openai base
44 | ```python 
45 | import openai 
46 | openai.api_base = "http://0.0.0.0:8000"
47 | 
48 | # cohere call
49 | response = openai.ChatCompletion.create(
50 |         model="command-nightly",
51 |         messages=[{"role":"user", "content":"Say this is a test!"}],
52 |         api_key = "your-cohere-api-key"
53 | )
54 | 
55 | # bedrock call
56 | response = openai.ChatCompletion.create(
57 |         model = "bedrock/anthropic.claude-instant-v1",
58 |         messages=[{"role":"user", "content":"Say this is a test!"}],
59 |         aws_access_key_id="",
60 |         aws_secret_access_key="",
61 |         aws_region_name="us-west-2",
62 | )
63 | 
64 | print(response)
65 | ``` 
66 | 
67 | [**See how to call Huggingface,Bedrock,TogetherAI,Anthropic, etc.**](https://docs.litellm.ai/docs/simple_proxy)
68 | 


--------------------------------------------------------------------------------
/openai-proxy/requirements.txt:
--------------------------------------------------------------------------------
1 | openai
2 | fastapi
3 | uvicorn
4 | boto3
5 | litellm


--------------------------------------------------------------------------------
/openai-proxy/tests/test_bedrock.py:
--------------------------------------------------------------------------------
 1 | import openai
 2 | openai.api_base = "http://0.0.0.0:8000"
 3 | print("making request")
 4 | openai.api_key = "anything" # this gets passed as a header 
 5 | 
 6 | 
 7 | response = openai.ChatCompletion.create(
 8 |     model = "bedrock/anthropic.claude-instant-v1",
 9 |     messages = [
10 |         {
11 |             "role": "user",
12 |             "content": "this is a test message, what model / llm are you"
13 |         }
14 |     ],
15 |     aws_access_key_id="",
16 |     aws_secret_access_key="",
17 |     aws_region_name="us-west-2",
18 |     max_tokens = 10,
19 | )
20 | 
21 | 
22 | print(response)
23 | 
24 | 
25 | # response = openai.ChatCompletion.create(
26 | #     model = "gpt-3.5-turbo",
27 | #     messages = [
28 | #         {
29 | #             "role": "user",
30 | #             "content": "this is a test message, what model / llm are you"
31 | #         }
32 | #     ],
33 | #     max_tokens = 10,
34 | #     stream=True
35 | # )
36 | 
37 | 
38 | # for chunk in response:
39 | #     print(chunk)


--------------------------------------------------------------------------------
/openai-proxy/tests/test_openai.py:
--------------------------------------------------------------------------------
 1 | import openai
 2 | openai.api_base = "http://127.0.0.1:8000"
 3 | openai.api_key = "this can be anything"
 4 | 
 5 | print("making request")
 6 | 
 7 | api_key = "" 
 8 | response = openai.ChatCompletion.create(
 9 |     model = "gpt-3.5-turbo",
10 |     messages = [
11 |         {
12 |             "role": "user",
13 |             "content": "this is a test message, what model / llm are you"
14 |         }
15 |     ],
16 |     api_key=api_key,
17 |     max_tokens = 10,
18 | )
19 | 
20 | 
21 | print(response)
22 | 
23 | 
24 | response = openai.ChatCompletion.create(
25 |     model = "gpt-3.5-turbo",
26 |     messages = [
27 |         {
28 |             "role": "user",
29 |             "content": "this is a test message, what model / llm are you"
30 |         }
31 |     ],
32 |     api_key=api_key,
33 |     max_tokens = 10,
34 |     stream=True
35 | )
36 | 
37 | 
38 | for chunk in response:
39 |     print(chunk)


--------------------------------------------------------------------------------
/openai-proxy/tests/test_openrouter.py:
--------------------------------------------------------------------------------
 1 | import openai
 2 | openai.api_base = "http://0.0.0.0:8000"
 3 | openai.api_key = "this can be anything"
 4 | print("making request")
 5 | 
 6 | api_key = ""
 7 | response = openai.ChatCompletion.create(
 8 |     model = "openrouter/google/palm-2-chat-bison",
 9 |     messages = [
10 |         {
11 |             "role": "user",
12 |             "content": "this is a test message, what model / llm are you"
13 |         }
14 |     ],
15 |     api_key=api_key,
16 |     max_tokens = 10,
17 | )
18 | 
19 | 
20 | print(response)
21 | 
22 | 
23 | response = openai.ChatCompletion.create(
24 |     model = "openrouter/google/palm-2-chat-bison",
25 |     messages = [
26 |         {
27 |             "role": "user",
28 |             "content": "this is a test message, what model / llm are you"
29 |         }
30 |     ],
31 |     api_key=api_key,
32 |     max_tokens = 10,
33 |     stream=True
34 | )
35 | 
36 | 
37 | for chunk in response:
38 |     print(chunk)


--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
 1 | [tool.poetry]
 2 | name = "litellm"
 3 | version = "0.12.2"
 4 | description = "Library to easily interface with LLM API providers"
 5 | authors = ["BerriAI"]
 6 | license = "MIT License"
 7 | readme = "README.md"
 8 | 
 9 | [tool.poetry.dependencies]
10 | python = "^3.8"
11 | openai = "^0.28.0 || ^0.27.0"
12 | python-dotenv = ">=0.2.0"
13 | tiktoken = ">=0.4.0"
14 | importlib-metadata = ">=6.8.0"
15 | tokenizers = "*"
16 | click = "*"
17 | jinja2 = "^3.1.2"
18 | certifi = "^2023.7.22"
19 | appdirs = "^1.4.4"
20 | 
21 | [tool.poetry.scripts]
22 | litellm = 'litellm:run_server'
23 | 
24 | [build-system]
25 | requires = ["poetry-core"]
26 | build-backend = "poetry.core.masonry.api"
27 | 
28 | [tool.commitizen]
29 | version = "0.12.2"
30 | version_files = [
31 |     "pyproject.toml:^version"
32 | ]
33 | 
34 | 


--------------------------------------------------------------------------------
/render.yaml:
--------------------------------------------------------------------------------
 1 | services:
 2 |   # A Docker web service
 3 |   - type: web
 4 |     name: webdis
 5 |     runtime: docker
 6 |     rootDir: ./
 7 |     healthCheckPath: /
 8 |     envVars:
 9 |     - key: PORT
10 |       value: 8000
11 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | openai
2 | fastapi
3 | uvicorn
4 | boto3
5 | litellm


--------------------------------------------------------------------------------