├── .dockerignore ├── .git-blame-ignore-revs ├── .gitattributes ├── .github ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.cn.md ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE │ ├── bug_report.yml │ ├── config.yml │ └── feature_request.yml ├── README.cn.md ├── README.jp.md ├── SECURITY.md ├── SUPPORT.md ├── cn.CODE_OF_CONDUCT.md ├── pull_request_template.md └── workflows │ ├── check_code_formatting.yml │ ├── docker_publish.yml │ ├── link-checker.yml │ ├── npm_publish.yml │ ├── run_tests.yml │ └── triage-label.yml ├── .gitignore ├── .husky ├── pre-commit └── pre-push ├── .prettierignore ├── .prettierrc ├── .vscode ├── launch.json └── tasks.json ├── Dockerfile ├── LICENSE ├── README.md ├── conf.json ├── conf_sample.json ├── cookbook ├── README.md ├── getting-started │ ├── automatic-retries-on-failures.md │ ├── enable-cache.md │ ├── fallback-from-openai-to-azure.ipynb │ ├── fallback-from-stable-diffusion-to-dall-e.ipynb │ ├── gentle-introduction-to-portkey-gateway.ipynb │ ├── image-generation.ipynb │ ├── resilient-loadbalancing-with-failure-mitigating-fallbacks.md │ └── writing-your-first-gateway-config.md ├── guardrails │ └── Langchain Chatbot with PII Guardrails.ipynb ├── integrations │ ├── Instructor_with_Portkey.ipynb │ ├── Mistral.ipynb │ ├── Phidata_with_ Perplexity.ipynb │ ├── Phidata_with_Portkey.ipynb │ ├── ReAct Agents Portkey.ipynb │ ├── Tool_Use_with_Portkey.ipynb │ ├── anyscale.md │ ├── deepinfra.ipynb │ ├── groq.ipynb │ ├── langchain.ipynb │ ├── mistral.md │ ├── mixtral-8x22b.ipynb │ ├── ncompass.ipynb │ ├── openai.ipynb │ ├── segmind.ipynb │ ├── vercel-ai.md │ └── vercel │ │ ├── app │ │ ├── api │ │ │ └── chat │ │ │ │ └── route.ts │ │ ├── examples │ │ │ ├── basic-chatbot │ │ │ │ └── page.tsx │ │ │ ├── generate-text │ │ │ │ ├── action.ts │ │ │ │ ├── conditional-routing.ts │ │ │ │ ├── fallback.ts │ │ │ │ ├── guardrails.ts │ │ │ │ ├── load-balance.ts │ │ │ │ └── page.tsx │ │ │ ├── generate-ui-streamui │ │ │ │ ├── action.tsx │ │ │ │ ├── joke-component.tsx │ │ │ │ ├── joke.ts │ │ │ │ └── page.tsx │ │ │ ├── layout.tsx │ │ │ ├── stream-text │ │ │ │ ├── action.ts │ │ │ │ └── page.tsx │ │ │ └── tools │ │ │ │ └── basic │ │ │ │ ├── action.ts │ │ │ │ └── page.tsx │ │ ├── favicon.ico │ │ ├── globals.css │ │ ├── layout.tsx │ │ └── page.tsx │ │ ├── components.json │ │ ├── components │ │ ├── back-button.tsx │ │ ├── link.tsx │ │ └── ui │ │ │ ├── button.tsx │ │ │ ├── card.tsx │ │ │ ├── input.tsx │ │ │ └── label.tsx │ │ ├── core │ │ ├── generate-text.ts │ │ ├── stream-text.ts │ │ └── tools.ts │ │ ├── lib │ │ └── utils.ts │ │ ├── next-env.d.ts │ │ ├── next.config.mjs │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── pnpm-lock.yaml │ │ ├── postcss.config.js │ │ ├── public │ │ ├── next.svg │ │ └── vercel.svg │ │ ├── readme.md │ │ ├── tailwind.config.ts │ │ └── tsconfig.json ├── monitoring-agents │ ├── Autogen_with_Telemetry.ipynb │ ├── ControlFlow_with_Telemetry.ipynb │ ├── CrewAI_with_Telemetry.ipynb │ └── Llama_Agents_with_Telemetry.ipynb ├── providers │ ├── anthropic.ipynb │ ├── deepinfra.ipynb │ ├── groq.ipynb │ ├── mistral.ipynb │ ├── ncompass.ipynb │ ├── nvidia.ipynb │ ├── openai.ipynb │ ├── segmind.ipynb │ └── together.ipynb └── use-cases │ ├── Claude_3_5_Sonnet_vs_GPT_4o_Portkey.ipynb │ ├── Claude_3_5_Sonnet_with_Portkey.ipynb │ ├── Contextual Embeddings Guide Anthropic, Cohere, Voyage.ipynb │ ├── Creating_Artifacts_with_GPT_4o_.ipynb │ ├── GPT-4o & Portkey │ ├── Audio Processing - GPT-4o & Portkey.ipynb │ ├── GPT-4o vs Claude-3-Opus vs Gemini-1.5-Pro - Portkey.ipynb │ ├── Image Processing - GPT-4o & Portkey.ipynb │ ├── Intro_to_GPT4o_Portkey.ipynb │ └── Video Processing - GPT-4o & Portkey.ipynb │ ├── LMSYS Series │ ├── Comparision_of_Top_6_LMSYS_Vison_models_using_Portkey.ipynb │ ├── README.md │ └── comparing-top10-LMSYS-models-with-Portkey.ipynb │ ├── Nemotron_GPT_Finetuning_Portkey.ipynb │ ├── Testing_Top_Vision_Models.ipynb │ ├── llama-3-on-groq.ipynb │ ├── run-gateway-on-prompts-from-langchain-hub.md │ ├── smart-fallback-with-model-optimized-prompts.md │ ├── supabase-pgvector-and-portkey.md │ └── use-openai-sdk-with-portkey-prompt-templates.md ├── deployment.yaml ├── docker-compose.yaml ├── docs ├── deploy-on-replit.md ├── images │ ├── Sticker.png │ ├── anthropic.png │ ├── anyscale.png │ ├── azure.png │ ├── bard.png │ ├── cohere.png │ ├── cookbook-header.png │ ├── cookbooks │ │ ├── 101-configs-1.png │ │ ├── 101-configs-2.png │ │ ├── cache-1.png │ │ ├── cache-2.png │ │ ├── cache-3.png │ │ ├── fallback-trace-id.png │ │ ├── langchain-hub.png │ │ ├── logs.png │ │ ├── prompt-template.png │ │ ├── resilient-loadbalance-1.png │ │ ├── resilient-loadbalance.png │ │ ├── supabase-1.png │ │ ├── supabase-2.png │ │ ├── supabase-3.png │ │ ├── supabase-4.png │ │ ├── supabase-5.png │ │ ├── supabase-6.png │ │ └── supabase-7.png │ ├── demo.gif │ ├── gateway-border.png │ ├── header.png │ ├── header_new.png │ ├── localai.png │ ├── openai.png │ └── palm.png └── installation-deployments.md ├── eslint.config.js ├── jest.config.js ├── package-lock.json ├── package.json ├── patches ├── @types+async-retry+1.4.5.patch └── async-retry+1.3.3.patch ├── plugins ├── Contributing.md ├── README.md ├── acuvity │ ├── helper.test.ts │ ├── helper.ts │ ├── manifest.json │ ├── model.ts │ ├── scan.test.ts │ └── scan.ts ├── aporia │ ├── aporia.test.ts │ ├── manifest.json │ └── validateProject.ts ├── azure │ ├── azure.test.ts │ ├── contentSafety.ts │ ├── manifest.json │ ├── pii.ts │ ├── types.ts │ └── utils.ts ├── bedrock │ ├── bedrock.test.ts │ ├── index.ts │ ├── manifest.json │ ├── type.ts │ └── util.ts ├── build.ts ├── default │ ├── alllowercase.ts │ ├── alluppercase.ts │ ├── characterCount.ts │ ├── contains.ts │ ├── containsCode.ts │ ├── default.test.ts │ ├── endsWith.ts │ ├── jsonKeys.ts │ ├── jsonSchema.ts │ ├── jwt.ts │ ├── log.ts │ ├── manifest.json │ ├── modelWhitelist.ts │ ├── regexMatch.ts │ ├── sentenceCount.ts │ ├── validUrls.ts │ ├── webhook.ts │ └── wordCount.ts ├── exa │ ├── exa.test.ts │ ├── manifest.json │ └── online.ts ├── index.ts ├── lasso │ ├── README.md │ ├── classify.ts │ ├── lasso.test.ts │ └── manifest.json ├── mistral │ ├── index.ts │ ├── manifest.json │ └── mistral.test.ts ├── pangea │ ├── manifest.json │ ├── pangea.test.ts │ ├── pii.ts │ ├── textGuard.ts │ └── version.ts ├── panw-prisma-airs │ ├── intercept.ts │ ├── manifest.json │ └── panw.airs.test.ts ├── patronus │ ├── custom.ts │ ├── globals.ts │ ├── isConcise.ts │ ├── isHelpful.ts │ ├── isPolite.ts │ ├── manifest.json │ ├── noApologies.ts │ ├── noGenderBias.ts │ ├── noRacialBias.ts │ ├── patronus.test.ts │ ├── phi.ts │ ├── pii.ts │ ├── retrievalAnswerRelevance.ts │ └── toxicity.ts ├── pillar │ ├── globals.ts │ ├── manifest.json │ ├── pillar.test.ts │ ├── scanPrompt.ts │ └── scanResponse.ts ├── portkey │ ├── gibberish.ts │ ├── globals.ts │ ├── language.ts │ ├── manifest.json │ ├── moderateContent.ts │ ├── pii.ts │ └── portkey.test.ts ├── promptfoo │ ├── globals.ts │ ├── guard.ts │ ├── harm.ts │ ├── manifest.json │ ├── pii.ts │ ├── promptfoo.test.ts │ └── types.ts ├── promptsecurity │ ├── manifest.json │ ├── promptsecurity.test.ts │ ├── protectPrompt.ts │ ├── protectResponse.ts │ └── shared.ts ├── sydelabs │ ├── manifest.json │ ├── sydeguard.ts │ └── sydelabs.test.ts ├── types.ts ├── utils.test.ts └── utils.ts ├── rollup.config.js ├── src ├── data │ ├── models.json │ └── providers.json ├── errors │ ├── GatewayError.ts │ └── RouterError.ts ├── globals.ts ├── handlers │ ├── batchesHandler.ts │ ├── chatCompletionsHandler.ts │ ├── completionsHandler.ts │ ├── createSpeechHandler.ts │ ├── createTranscriptionHandler.ts │ ├── createTranslationHandler.ts │ ├── embeddingsHandler.ts │ ├── filesHandler.ts │ ├── finetuneHandler.ts │ ├── handlerUtils.ts │ ├── imageGenerationsHandler.ts │ ├── modelResponsesHandler.ts │ ├── modelsHandler.ts │ ├── proxyHandler.ts │ ├── realtimeHandler.ts │ ├── realtimeHandlerNode.ts │ ├── responseHandlers.ts │ ├── retryHandler.ts │ ├── streamHandler.ts │ ├── streamHandlerUtils.ts │ └── websocketUtils.ts ├── index.ts ├── middlewares │ ├── cache │ │ └── index.ts │ ├── hooks │ │ ├── globals.ts │ │ ├── index.ts │ │ └── types.ts │ ├── log │ │ └── index.ts │ └── requestValidator │ │ ├── index.ts │ │ └── schema │ │ └── config.ts ├── providers │ ├── ai21 │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── anthropic │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── index.ts │ │ └── types.ts │ ├── anyscale │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── azure-ai-inference │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── azure-openai │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── createBatch.ts │ │ ├── createFinetune.ts │ │ ├── createSpeech.ts │ │ ├── createTranscription.ts │ │ ├── createTranslation.ts │ │ ├── embed.ts │ │ ├── getBatchOutput.ts │ │ ├── imageGenerate.ts │ │ ├── index.ts │ │ ├── uploadFile.ts │ │ └── utils.ts │ ├── bedrock │ │ ├── api.ts │ │ ├── cancelBatch.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── constants.ts │ │ ├── createBatch.ts │ │ ├── createFinetune.ts │ │ ├── deleteFile.ts │ │ ├── embed.ts │ │ ├── getBatchOutput.ts │ │ ├── imageGenerate.ts │ │ ├── index.ts │ │ ├── listBatches.ts │ │ ├── listFinetunes.ts │ │ ├── listfiles.ts │ │ ├── retrieveBatch.ts │ │ ├── retrieveFile.ts │ │ ├── retrieveFileContent.ts │ │ ├── retrieveFinetune.ts │ │ ├── types.ts │ │ ├── uploadFile.ts │ │ ├── uploadFileUtils.ts │ │ └── utils.ts │ ├── cerebras │ │ ├── api.ts │ │ └── index.ts │ ├── cohere │ │ ├── api.ts │ │ ├── cancelBatch.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── createBatch.ts │ │ ├── deleteFile.ts │ │ ├── embed.ts │ │ ├── getBatchOutput.ts │ │ ├── getFiles.ts │ │ ├── index.ts │ │ ├── listBatches.ts │ │ ├── retrieveBatch.ts │ │ ├── types.ts │ │ ├── uploadFile.ts │ │ └── utils.ts │ ├── cortex │ │ ├── api.ts │ │ └── index.ts │ ├── dashscope │ │ ├── api.ts │ │ └── index.ts │ ├── deepbricks │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── imageGenerate.ts │ │ └── index.ts │ ├── deepinfra │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── deepseek │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── fireworks-ai │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── embed.ts │ │ ├── imageGenerate.ts │ │ ├── index.ts │ │ ├── listFiles.ts │ │ ├── retrieveFile.ts │ │ ├── types.ts │ │ ├── uploadFile.ts │ │ └── utils.ts │ ├── google-vertex-ai │ │ ├── api.ts │ │ ├── cancelBatch.ts │ │ ├── chatComplete.ts │ │ ├── createBatch.ts │ │ ├── createFinetune.ts │ │ ├── embed.ts │ │ ├── getBatchOutput.ts │ │ ├── imageGenerate.ts │ │ ├── index.ts │ │ ├── listBatches.ts │ │ ├── listFiles.ts │ │ ├── listFinetunes.ts │ │ ├── retrieveBatch.ts │ │ ├── retrieveFile.ts │ │ ├── retrieveFileContent.ts │ │ ├── retrieveFinetune.ts │ │ ├── transformGenerationConfig.ts │ │ ├── types.ts │ │ ├── uploadFile.ts │ │ └── utils.ts │ ├── google │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── groq │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── huggingface │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── index.ts │ │ ├── types.ts │ │ └── utils.ts │ ├── index.ts │ ├── inference-net │ │ ├── api.ts │ │ └── index.ts │ ├── jina │ │ ├── api.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── lambda │ │ ├── api.ts │ │ └── index.ts │ ├── lemonfox-ai │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── createTranscription.ts │ │ ├── imageGenerate.ts │ │ └── index.ts │ ├── lepton │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── createTranscription.ts │ │ └── index.ts │ ├── lingyi │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── milvus │ │ ├── api.ts │ │ └── index.ts │ ├── mistral-ai │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── monsterapi │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── moonshot │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── ncompass │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── nebius │ │ ├── api.ts │ │ └── index.ts │ ├── nomic │ │ ├── api.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── novita-ai │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ └── index.ts │ ├── nscale │ │ ├── api.ts │ │ ├── imageGenerate.ts │ │ └── index.ts │ ├── ollama │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── open-ai-base │ │ ├── constants.ts │ │ ├── createModelResponse.ts │ │ ├── helpers.ts │ │ └── index.ts │ ├── openai │ │ ├── api.ts │ │ ├── cancelBatch.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── createBatch.ts │ │ ├── createFinetune.ts │ │ ├── createSpeech.ts │ │ ├── createTranscription.ts │ │ ├── createTranslation.ts │ │ ├── deleteFile.ts │ │ ├── embed.ts │ │ ├── getBatchOutput.ts │ │ ├── imageGenerate.ts │ │ ├── index.ts │ │ ├── listBatches.ts │ │ ├── listFiles.ts │ │ ├── retrieveBatch.ts │ │ ├── retrieveFileContent.ts │ │ ├── uploadFile.ts │ │ └── utils.ts │ ├── openrouter │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── palm │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── perplexity-ai │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── predibase │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── qdrant │ │ ├── api.ts │ │ └── index.ts │ ├── recraft-ai │ │ ├── api.ts │ │ ├── imageGenerate.ts │ │ ├── index.ts │ │ └── utils.ts │ ├── reka-ai │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── replicate │ │ ├── api.ts │ │ └── index.ts │ ├── sagemaker │ │ ├── api.ts │ │ └── index.ts │ ├── sambanova │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ └── index.ts │ ├── segmind │ │ ├── api.ts │ │ ├── imageGenerate.ts │ │ └── index.ts │ ├── siliconflow │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── embed.ts │ │ ├── imageGenerate.ts │ │ └── index.ts │ ├── stability-ai │ │ ├── api.ts │ │ ├── constants.ts │ │ ├── imageGenerate.ts │ │ ├── imageGenerateV2.ts │ │ ├── index.ts │ │ └── utils.ts │ ├── together-ai │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── triton │ │ ├── api.ts │ │ ├── complete.ts │ │ └── index.ts │ ├── types.ts │ ├── upstage │ │ ├── api.ts │ │ └── index.ts │ ├── utils.ts │ ├── voyage │ │ ├── api.ts │ │ ├── embed.ts │ │ └── index.ts │ ├── workers-ai │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── complete.ts │ │ ├── embed.ts │ │ ├── imageGenerate.ts │ │ ├── index.ts │ │ └── utils.ts │ ├── x-ai │ │ ├── api.ts │ │ └── index.ts │ └── zhipu │ │ ├── api.ts │ │ ├── chatComplete.ts │ │ ├── embed.ts │ │ └── index.ts ├── public │ └── index.html ├── services │ ├── conditionalRouter.ts │ ├── realtimeLlmEventParser.ts │ └── transformToProviderRequest.ts ├── start-server.ts ├── tests │ ├── common.test.ts │ ├── resources │ │ ├── constants.ts │ │ ├── requestTemplates.ts │ │ ├── testVariables.ts │ │ └── utils.ts │ └── routeSpecificTestFunctions.ts │ │ └── chatCompletion.ts ├── types │ ├── embedRequestBody.ts │ ├── inputList.ts │ ├── modelResponses.ts │ ├── requestBody.ts │ ├── responseBody.ts │ └── shared.ts ├── utils.ts └── utils │ └── misc.ts ├── start-test.js ├── tsconfig.json └── wrangler.toml /.dockerignore: -------------------------------------------------------------------------------- 1 | .toml 2 | docs -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Initial prettier formatting 2 | 77f32e57b14bd3585169b7bc884ff9ddf38e8ea1 -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Declare files that will always have LF line endings on checkout. 2 | *.sh text eol=lf 3 | 4 | # Denote all files that are truly binary and should not be modified. 5 | *.png binary 6 | *.jpg binary 7 | *.jpeg binary 8 | *.gif binary 9 | *.ico binary 10 | *.mov binary 11 | *.mp4 binary 12 | *.mp3 binary 13 | *.flv binary 14 | *.fla binary 15 | *.swf binary 16 | *.gz binary 17 | *.zip binary 18 | *.7z binary 19 | *.ttf binary 20 | *.eot binary 21 | *.woff binary 22 | *.woff2 binary 23 | 24 | # Ignore cookbook folder for language statistics 25 | cookbook/** linguist-vendored 26 | docs/** linguist-vendored 27 | 28 | # Treat package-lock.json and yarn.lock as binary to avoid merge conflicts 29 | package-lock.json binary 30 | yarn.lock binary 31 | 32 | # Treat .env files as binary to avoid accidental commits of sensitive information 33 | *.env binary -------------------------------------------------------------------------------- /.github/CONTRIBUTING.cn.md: -------------------------------------------------------------------------------- 1 | ## 🎉 欢迎 2 | 你好，感谢你考虑为Portkey的AI网关做出贡献！无论你是在报告一个bug，建议一个功能，改进文档，还是编写代码，你的贡献对我们来说都是非常宝贵的。 3 | 4 | ## 🚀 快速开始 5 | 1. 在Github上Fork仓库。 6 | 2. 将你fork的仓库克隆到你的机器上。 7 | ```sh 8 | $ git clone https://github.com/YOUR_USERNAME/gateway.git 9 | ``` 10 | 11 | ## 🖋 贡献类型 12 | 1. 新的集成：为其他LLM供应商或总体供应商创建集成。 13 | 2. Bug修复 14 | 3. 增强 15 | 4. 文档 16 | 5. **Hacktoberfest** 提交！ 17 | 18 | 19 | ## 🗓️ Hacktoberfest 20 | 在 [Hacktoberfest 月](https://hacktoberfest.com/)期间，从10月1日到31日，您被接受的 PR 将计入您的 Hacktoberfest 参与度！🚀 21 | ✅ 要获得接受，您的 PR 必须被合并、批准或贴上 `hacktoberfest-accepted` 标签。 22 | 🧐 记得遵守 [质量标准](https://hacktoberfest.digitalocean.com/resources/qualitystandards)以避免您的 PR 被标记为 `垃圾邮件` 或 `无效`。 23 | ## 🔄 提交 PR 24 | 1. 完成您的更改后，通过运行以下命令来格式化和 Lint 您的代码： 25 | ```sh 26 | make format; make lint 27 | ``` 28 | 2. 在提交 PR 时，请在标题前添加以下任何一个： 29 | * `provider:` 添加新的 LLM 提供商。 30 | * `integration:` 新的集成。 31 | * `docs:` 新的指南、文档添加等。 32 | * `improvement:` 改进或增强。 33 | * `bug:` 修复错误。 34 | * `hacktoberfest:` Hacktoberfest 贡献。 35 | ## 🤔 寻求帮助 36 | 遇到问题或有疑问？请毫不犹豫地在我们的 [Discord 社区](https://discord.com/invite/DD7vgKK299) 分享您的疑问或问题 - 这是获得支持和与其他贡献者联系的最快方式。 37 | ## 🚧 发布流程 38 | 我们尽快进行发布，以确保新功能和修复能快速地到达用户手中。我们遵循无缝的 CI/CD 流水线，以确保代码从开发到生产的平稳过渡。 39 | ## 🎊 您的 PR 被合并了！ 40 | 所有成功的 PR 都会在我们的 [Discord](https://discord.com/invite/DD7vgKK299) 上庆祝，并在发布说明中提及，重大贡献将在我们的 [Twitter](https://twitter.com/PortkeyAI) 上突出显示。请继续关注，未来我们将为贡献者提供更多奖金和礼品！ -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: Report any issue with the project 3 | labels: ["bug"] 4 | body: 5 | - type: textarea 6 | id: what-happened 7 | attributes: 8 | label: What Happened? 9 | validations: 10 | required: true 11 | - type: textarea 12 | id: expected-behavior 13 | attributes: 14 | label: What Should Have Happened? 15 | validations: 16 | required: false 17 | - type: textarea 18 | id: code-snippet 19 | attributes: 20 | label: Relevant Code Snippet 21 | validations: 22 | required: false 23 | - type: input 24 | id: contact 25 | attributes: 26 | label: Your Twitter/LinkedIn 27 | description: When the bug get fixed, we'd like to thank you publicly for reporting it. 28 | validations: 29 | required: false 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | contact_links: 3 | - name: Discord for Support & Discussions 4 | url: https://discord.com/invite/DD7vgKK299 5 | about: Hang out with the community of LLM practitioners and resolve your issues fast 6 | - name: Get on a Call 7 | url: https://calendly.com/rohit-portkey/noam 8 | about: Get a tailored demo for your use cases 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.yml: -------------------------------------------------------------------------------- 1 | name: Feature Request 2 | description: Suggest a new provider to integrate, new features, or something more 3 | title: "[Feature] " 4 | labels: ["enhancement"] 5 | body: 6 | - type: textarea 7 | id: feature 8 | attributes: 9 | label: What Would You Like to See with the Gateway? 10 | validations: 11 | required: true 12 | - type: textarea 13 | id: context 14 | attributes: 15 | label: Context for your Request 16 | description: Why you want this feature and how it beneits you. 17 | validations: 18 | required: false 19 | - type: input 20 | id: contact 21 | attributes: 22 | label: Your Twitter/LinkedIn 23 | description: If we work on this request, we'd like to thank you publicly for suggesting it. 24 | validations: 25 | required: false 26 | -------------------------------------------------------------------------------- /.github/SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported | 6 | | ------- | ------------------ | 7 | | 1.x.x | :white_check_mark: | 8 | 9 | ## Reporting a Vulnerability 10 | 11 | Please report any security vulnerabilities at support@portkey.ai 12 | -------------------------------------------------------------------------------- /.github/SUPPORT.md: -------------------------------------------------------------------------------- 1 | ## How to file issues and get help 2 | 3 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing issues before filing new issues to avoid duplicates. 4 | For new issues, file your bug or feature request as a new Issue. 5 | 6 | For help and questions about using this project, please contact `support@portkey.ai`. Join the community discussions [here](https://discord.com/invite/DD7vgKK299). 7 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | 4 | ## Motivation 5 | 6 | 7 | ## Type of Change 8 | 9 | - [ ] Bug fix (non-breaking change which fixes an issue) 10 | - [ ] New feature (non-breaking change which adds functionality) 11 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 12 | - [ ] Documentation update 13 | - [ ] Refactoring (no functional changes) 14 | 15 | ## How Has This Been Tested? 16 | 17 | - [ ] Unit Tests 18 | - [ ] Integration Tests 19 | - [ ] Manual Testing 20 | 21 | ## Screenshots (if applicable) 22 | 23 | 24 | ## Checklist 25 | 26 | - [ ] My code follows the style guidelines of this project 27 | - [ ] I have performed a self-review of my own code 28 | - [ ] I have commented my code, particularly in hard-to-understand areas 29 | - [ ] I have made corresponding changes to the documentation 30 | - [ ] My changes generate no new warnings 31 | - [ ] I have added tests that prove my fix is effective or that my feature works 32 | - [ ] New and existing unit tests pass locally with my changes 33 | 34 | ## Related Issues 35 | 36 | -------------------------------------------------------------------------------- /.github/workflows/check_code_formatting.yml: -------------------------------------------------------------------------------- 1 | name: Check Prettier Formatting 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | check-formatting: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v2 15 | 16 | - name: Setup Node.js 17 | uses: actions/setup-node@v2 18 | with: 19 | node-version: '20.x' 20 | 21 | - name: Install dependencies 22 | run: npm install 23 | 24 | - name: Check formatting 25 | run: npm run format:check 26 | -------------------------------------------------------------------------------- /.github/workflows/docker_publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | push_to_registry: 9 | name: Push Docker image to Docker Hub 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Check out the repo 13 | uses: actions/checkout@v4 14 | 15 | - name: Set up QEMU 16 | uses: docker/setup-qemu-action@v3 17 | 18 | - name: Set up Docker Buildx 19 | uses: docker/setup-buildx-action@v3 20 | 21 | - name: Log in to Docker Hub 22 | uses: docker/login-action@v3 23 | with: 24 | username: ${{ secrets.DOCKER_USERNAME }} 25 | password: ${{ secrets.DOCKER_PASSWORD }} 26 | 27 | - name: Extract metadata (tags, labels) for Docker 28 | id: meta 29 | uses: docker/metadata-action@v5 30 | with: 31 | images: ${{ secrets.DOCKER_ORGANISATION }}/gateway 32 | tags: | 33 | type=raw,value=latest 34 | type=pep440,pattern={{version}},value=${{ github.event.release.tag_name }} 35 | 36 | - name: Build and push Docker image 37 | uses: docker/build-push-action@v5 38 | with: 39 | context: . 40 | file: ./Dockerfile 41 | platforms: linux/amd64,linux/arm64 42 | push: true 43 | tags: ${{ steps.meta.outputs.tags }} 44 | labels: ${{ steps.meta.outputs.labels }} 45 | -------------------------------------------------------------------------------- /.github/workflows/npm_publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to NPM 2 | on: 3 | release: 4 | types: [published] 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout 10 | uses: actions/checkout@v4 11 | - name: Setup Node 12 | uses: actions/setup-node@v3 13 | with: 14 | node-version: '20.x' 15 | registry-url: 'https://registry.npmjs.org' 16 | - name: Install dependencies 17 | run: npm ci 18 | - name: Publish package on NPM 19 | run: npm publish 20 | env: 21 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/triage-label.yml: -------------------------------------------------------------------------------- 1 | name: Auto Triage Label 2 | 3 | on: 4 | issues: 5 | types: [opened] 6 | 7 | jobs: 8 | label_issues: 9 | runs-on: ubuntu-latest 10 | 11 | permissions: 12 | issues: write 13 | 14 | steps: 15 | - uses: actions/checkout@v4.1.1 16 | - name: Add Triage Label 17 | uses: actions/github-script@v7.0.1 18 | with: 19 | script: | 20 | github.rest.issues.addLabels({ 21 | issue_number: context.issue.number, 22 | owner: context.repo.owner, 23 | repo: context.repo.repo, 24 | labels: ['triage'] 25 | }) 26 | github-token: ${{secrets.GITHUB_TOKEN}} 27 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | npm run format:check || (npm run format && return 1) 2 | -------------------------------------------------------------------------------- /.husky/pre-push: -------------------------------------------------------------------------------- 1 | npm run pre-push -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | build 4 | .wrangler 5 | public 6 | wrangler.toml -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": true, 3 | "tabWidth": 2, 4 | "printWidth": 80, 5 | "endOfLine": "lf", 6 | "singleQuote": true, 7 | "arrowParens": "always", 8 | "bracketSpacing": true, 9 | "trailingComma": "es5" 10 | } 11 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | /* 3 | 1. steps to run wrangler debugger (https://blog.cloudflare.com/debugging-cloudflare-workers/) 4 | - run `npm run dev` in a terminal 5 | - in debug and run tab, select 'Wrangler debug' and run 6 | - add a breakpoint and verify 7 | 8 | 2. Steps tp run node debugger 9 | - In debug tab select "Node debug" and run 10 | - add a breakpoint and verify 11 | */ 12 | "version": "0.2.0", 13 | "configurations": [ 14 | { 15 | "name": "Wrangler debug", 16 | "type": "node", 17 | "request": "attach", 18 | "port": 9229, 19 | "cwd": "/", 20 | "resolveSourceMapLocations": null, 21 | "attachExistingChildren": false, 22 | "autoAttachChildProcesses": false 23 | }, 24 | { 25 | "type": "node", 26 | "request": "launch", 27 | "name": "Node debug", 28 | "runtimeExecutable": "tsx", 29 | "runtimeArgs": ["${workspaceFolder}/build/src/start-server.js"], 30 | "sourceMaps": true, 31 | "outFiles": ["${workspaceFolder}/build/**/*.js"], 32 | "sourceMapPathOverrides": { 33 | "../src/*": "${workspaceFolder}/src/*" 34 | }, 35 | "skipFiles": ["/**"], 36 | "preLaunchTask": "npx tsc", 37 | "console": "integratedTerminal", 38 | "internalConsoleOptions": "neverOpen", 39 | "postDebugTask": "cleanup build" 40 | } 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "type": "shell", 8 | "label": "npx tsc", 9 | "command": "npx", 10 | "args": [ 11 | "tsc", 12 | "--sourcemap", 13 | "true", 14 | "--outDir", 15 | "${workspaceFolder}/build" 16 | ], 17 | "problemMatcher": ["$tsc"], 18 | "group": { 19 | "kind": "build", 20 | "isDefault": true 21 | } 22 | }, 23 | { 24 | "type": "shell", 25 | "label": "cleanup build", 26 | "command": "rm", 27 | "args": ["-rf", "${workspaceFolder}/build"], 28 | "problemMatcher": [], 29 | "group": "build", 30 | "presentation": { 31 | "reveal": "always", 32 | "panel": "new" 33 | } 34 | } 35 | ] 36 | } 37 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the official Node.js runtime as a parent image 2 | FROM node:20-alpine AS build 3 | 4 | # Set the working directory in the container 5 | WORKDIR /app 6 | 7 | # Copy package.json and package-lock.json to the working directory 8 | COPY package*.json ./ 9 | COPY patches ./ 10 | 11 | # Upgrade system packages 12 | RUN apk update && apk upgrade --no-cache 13 | 14 | # Upgrade npm to version 10.9.2 15 | RUN npm install -g npm@10.9.2 16 | 17 | # Install app dependencies 18 | RUN npm install 19 | 20 | # Copy the rest of the application code 21 | COPY . . 22 | 23 | # Build the application and clean up 24 | RUN npm run build \ 25 | && rm -rf node_modules \ 26 | && npm install --omit=dev 27 | 28 | # Use the official Node.js runtime as a parent image 29 | FROM node:20-alpine 30 | 31 | # Upgrade system packages 32 | RUN apk update && apk upgrade --no-cache 33 | 34 | # Upgrade npm to version 10.9.2 35 | RUN npm install -g npm@10.9.2 36 | 37 | # Set the working directory in the container 38 | WORKDIR /app 39 | 40 | # Copy the build directory, node_modules, and package.json to the working directory 41 | COPY --from=build /app/build /app/build 42 | COPY --from=build /app/node_modules /app/node_modules 43 | COPY --from=build /app/package.json /app/package.json 44 | COPY --from=build /app/patches /app/patches 45 | 46 | # Expose port 8787 47 | EXPOSE 8787 48 | 49 | ENTRYPOINT ["npm"] 50 | CMD ["run", "start:node"] 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Portkey, Inc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "plugins_enabled": [ 3 | "default", 4 | "portkey", 5 | "aporia", 6 | "sydelabs", 7 | "pillar", 8 | "patronus", 9 | "pangea", 10 | "promptsecurity", 11 | "panw-prisma-airs" 12 | ], 13 | "credentials": { 14 | "portkey": { 15 | "apiKey": "..." 16 | } 17 | }, 18 | "cache": false 19 | } 20 | -------------------------------------------------------------------------------- /conf_sample.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/conf_sample.json -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/api/chat/route.ts: -------------------------------------------------------------------------------- 1 | import { streamText } from 'ai'; 2 | import { openai } from '@ai-sdk/openai'; 3 | 4 | export async function POST(request: Request) { 5 | const { messages } = await request.json(); 6 | const stream = await streamText({ 7 | model: openai('gpt-4o'), 8 | system: 'You are a helpful assistant.', 9 | messages, 10 | }); 11 | return stream.toAIStreamResponse(); 12 | } 13 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/basic-chatbot/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { useChat } from 'ai/react'; 4 | 5 | export default function Chat() { 6 | const { messages, input, handleInputChange, handleSubmit } = useChat(); 7 | return ( 8 |
9 | {messages.map((m) => ( 10 |
11 | {m.role === 'user' ? 'User: ' : 'AI: '} 12 | {m.content} 13 |
14 | ))} 15 |
16 | 21 |
22 |
23 | ); 24 | } 25 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/generate-text/action.ts: -------------------------------------------------------------------------------- 1 | 'use server'; 2 | 3 | import { generateText } from 'ai'; 4 | import { createPortkey } from '@portkey-ai/vercel-provider'; 5 | 6 | export const generateTextAction = async () => { 7 | const llmClient = createPortkey({ 8 | apiKey: 'PORTKEY_API_KEY', 9 | virtualKey: 'YOUR_OPENAI_VIRTUAL_KEY', //head over to https://app.portkey.ai to create Virtual key 10 | 11 | //Portkey's config allows you to use- loadbalance, fallback, retires, timeouts, semantic caching, conditional routing, guardrails,etc. Head over to portkey docs to learn more 12 | }); 13 | 14 | // Learn more at docs.portkey.ai 15 | 16 | const result = await generateText({ 17 | model: llmClient.completionModel('gpt-3.5-turbo'), //choose model of choice 18 | prompt: 'tell me a joke', 19 | }); 20 | 21 | return result.text; 22 | }; 23 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/generate-text/fallback.ts: -------------------------------------------------------------------------------- 1 | 'use server'; 2 | 3 | import { generateText } from 'ai'; 4 | import { createPortkey } from '@portkey-ai/vercel-provider'; 5 | 6 | export const generateTextAction = async () => { 7 | // Fallback config 8 | const portkey_config = { 9 | strategy: { 10 | mode: 'fallback', 11 | }, 12 | targets: [ 13 | { 14 | provider: 'anthropic', 15 | api_key: 'YOUR_ANTHROPIC_API_KEY', 16 | override_params: { 17 | model: 'claude-3-5-sonnet-20240620', 18 | }, 19 | }, 20 | { 21 | provider: 'openai', 22 | api_key: 'YOUR_OPENAI_API_KEY', 23 | override_params: { 24 | model: 'gpt-4o', 25 | }, 26 | }, 27 | ], 28 | }; 29 | 30 | const llmClient = createPortkey({ 31 | apiKey: 'PORTKEY_API_KEY', 32 | config: portkey_config, 33 | //Portkey's config allows you to use- loadbalance, fallback, retires, timeouts, semantic caching, conditional routing, guardrails,etc. Head over to portkey docs to learn more 34 | //we are using API keys inside config, that's why no virtual keys needed 35 | }); 36 | 37 | // Learn more at docs.portkey.ai 38 | 39 | const result = await generateText({ 40 | model: llmClient.completionModel('gpt-3.5-turbo'), //choose model of choice 41 | prompt: 'tell me a joke', 42 | }); 43 | 44 | return result.text; 45 | }; 46 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/generate-text/guardrails.ts: -------------------------------------------------------------------------------- 1 | 'use server'; 2 | 3 | import { generateText } from 'ai'; 4 | import { createPortkey } from '@portkey-ai/vercel-provider'; 5 | 6 | export const generateTextAction = async () => { 7 | const portkey_config = { 8 | retry: { 9 | attempts: 3, 10 | }, 11 | cache: { 12 | mode: 'simple', 13 | }, 14 | virtual_key: 'openai-xxx', 15 | before_request_hooks: [ 16 | { 17 | id: 'input-guardrail-id-xx', 18 | }, 19 | ], 20 | after_request_hooks: [ 21 | { 22 | id: 'output-guardrail-id-xx', 23 | }, 24 | ], 25 | }; 26 | 27 | const llmClient = createPortkey({ 28 | apiKey: 'PORTKEY_API_KEY', 29 | config: portkey_config, 30 | //Portkey's config allows you to use- loadbalance, fallback, retires, timeouts, semantic caching, conditional routing, guardrails,etc. Head over to portkey docs to learn more 31 | //we are using API keys inside config, that's why no virtual keys needed 32 | }); 33 | 34 | // Learn more at docs.portkey.ai 35 | 36 | const result = await generateText({ 37 | model: llmClient.completionModel('gpt-3.5-turbo'), //choose model of choice 38 | prompt: 'tell me a joke', 39 | }); 40 | 41 | return result.text; 42 | }; 43 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/generate-text/load-balance.ts: -------------------------------------------------------------------------------- 1 | 'use server'; 2 | 3 | import { generateText } from 'ai'; 4 | import { createPortkey } from '@portkey-ai/vercel-provider'; 5 | 6 | export const generateTextAction = async () => { 7 | // Fallback config 8 | const portkey_config = { 9 | strategy: { 10 | mode: 'loadbalance', 11 | }, 12 | targets: [ 13 | { 14 | provider: 'anthropic', 15 | api_key: 'YOUR_ANTHROPIC_API_KEY', 16 | override_params: { 17 | model: 'claude-3-5-sonnet-20240620', 18 | }, 19 | weight: 0.25, 20 | }, 21 | { 22 | provider: 'openai', 23 | api_key: 'YOUR_OPENAI_API_KEY', 24 | override_params: { 25 | model: 'gpt-4o', 26 | }, 27 | weight: 0.75, 28 | }, 29 | ], 30 | }; 31 | 32 | const llmClient = createPortkey({ 33 | apiKey: 'PORTKEY_API_KEY', 34 | config: portkey_config, 35 | //Portkey's config allows you to use- loadbalance, fallback, retires, timeouts, semantic caching, conditional routing, guardrails,etc. Head over to portkey docs to learn more 36 | //we are using API keys inside config, that's why no virtual keys needed 37 | }); 38 | 39 | // Learn more at docs.portkey.ai 40 | 41 | const result = await generateText({ 42 | model: llmClient.completionModel('gpt-3.5-turbo'), //choose model of choice 43 | prompt: 'tell me a joke', 44 | }); 45 | 46 | return result.text; 47 | }; 48 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/generate-text/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Button } from '@/components/ui/button'; 4 | import { generateTextAction } from './action'; 5 | import { useState } from 'react'; 6 | 7 | export default function Page() { 8 | const [generation, setGeneration] = useState(''); 9 | return ( 10 |
11 |

Generate Text Example

12 | 20 | 
{JSON.stringify(generation, null, 2)}
21 |
22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/generate-ui-streamui/joke-component.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { useState } from 'react'; 4 | import { Button } from '@/components/ui/button'; 5 | import { Joke } from './joke'; 6 | 7 | export const JokeComponent = ({ joke }: { joke?: Joke }) => { 8 | const [showPunchline, setShowPunchline] = useState(false); 9 | return ( 10 |
11 |

{showPunchline ? joke?.punchline : joke?.setup}

12 | 19 |
20 | ); 21 | }; 22 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/generate-ui-streamui/joke.ts: -------------------------------------------------------------------------------- 1 | import { DeepPartial } from 'ai'; 2 | import { z } from 'zod'; 3 | 4 | export const jokeSchema = z.object({ 5 | setup: z.string().describe('the setup of the joke'), 6 | punchline: z.string().describe('the punchline of the joke'), 7 | }); 8 | 9 | export type Joke = DeepPartial; 10 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/layout.tsx: -------------------------------------------------------------------------------- 1 | export default function Layout({ children }: { children: React.ReactNode }) { 2 | return
{children}
; 3 | } 4 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/stream-text/action.ts: -------------------------------------------------------------------------------- 1 | 'use server'; 2 | 3 | import { streamText } from 'ai'; 4 | import { openai } from '@ai-sdk/openai'; 5 | import { createStreamableValue } from 'ai/rsc'; 6 | import { createPortkey } from '@portkey-ai/vercel-provider'; 7 | 8 | export const streamTextAction = async () => { 9 | const llmClient = createPortkey({ 10 | apiKey: 'PORTKEY_API_KEY', 11 | virtualKey: 'YOUR_OPENAI_VIRTUAL_KEY', 12 | }); 13 | 14 | const result = await streamText({ 15 | model: llmClient.completionModel('gpt-3.5-turbo-instruct'), 16 | temperature: 0.5, 17 | prompt: 'Tell me a joke.', 18 | }); 19 | return createStreamableValue(result.textStream).value; 20 | }; 21 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/stream-text/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Button } from '@/components/ui/button'; 4 | import { streamTextAction } from './action'; 5 | import { useState } from 'react'; 6 | import { readStreamableValue } from 'ai/rsc'; 7 | 8 | export default function Page() { 9 | const [generation, setGeneration] = useState(''); 10 | return ( 11 |
12 |

Stream Text Example

13 | 22 | 
{JSON.stringify(generation, null, 2)}
23 |
24 | ); 25 | } 26 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/examples/tools/basic/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Button } from '@/components/ui/button'; 4 | import { generateTextAction } from './action'; 5 | import { useState } from 'react'; 6 | import { Input } from '@/components/ui/input'; 7 | import { readStreamableValue } from 'ai/rsc'; 8 | 9 | export default function Page() { 10 | const [generation, setGeneration] = useState(''); 11 | return ( 12 |
13 |

Stream Text Tool Example

14 |
{ 16 | const location = data.get('location') as string; 17 | const result = await generateTextAction(location); 18 | if (result) { 19 | for await (const delta of readStreamableValue(result)) { 20 | setGeneration(delta ?? ''); 21 | } 22 | } 23 | }} 24 | > 25 | 26 | 27 |
28 | 
{generation}
29 |
30 | ); 31 | } 32 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/cookbook/integrations/vercel/app/favicon.ico -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import type { Metadata } from 'next'; 2 | import { Inter } from 'next/font/google'; 3 | import './globals.css'; 4 | import { AI } from './examples/generate-ui-streamui/action'; 5 | import { BackButton } from '@/components/back-button'; 6 | 7 | const inter = Inter({ subsets: ['latin'] }); 8 | 9 | export const metadata: Metadata = { 10 | title: 'Create Next App', 11 | description: 'Generated by create next app', 12 | }; 13 | 14 | export default function RootLayout({ 15 | children, 16 | }: Readonly<{ 17 | children: React.ReactNode; 18 | }>) { 19 | return ( 20 | 21 | 22 |
23 | 24 |
25 | 26 |
27 |
{children}
28 | 29 |
30 | 31 | 32 | ); 33 | } 34 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/app/page.tsx: -------------------------------------------------------------------------------- 1 | import { Link } from '@/components/link'; 2 | 3 | export default function Page() { 4 | return ( 5 |
6 |

7 | Vercel AI SDK Fundamentals with Portkey AI 8 |

9 |

10 | The following examples aim to showcase the fundamentals behind the 11 | Vercel AI SDK. The examples have minimal loading states to remain as 12 | simple as possible. 13 |

14 |

15 | The prompt for the first 2 examples (stream/generate text) is `Tell me a 16 | joke`. 17 |

18 |
    19 |
  • 20 | Generate Text 21 |
    • 22 |
  • 23 | Stream Text 24 |
    • 25 |
  • 26 | Basic Tool 27 |
    • 28 |
  • 29 | Chatbot with `useChat` 30 |
    • 31 |
32 |
33 | ); 34 | } 35 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/components.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://ui.shadcn.com/schema.json", 3 | "style": "default", 4 | "rsc": true, 5 | "tsx": true, 6 | "tailwind": { 7 | "config": "tailwind.config.ts", 8 | "css": "app/globals.css", 9 | "baseColor": "neutral", 10 | "cssVariables": true, 11 | "prefix": "" 12 | }, 13 | "aliases": { 14 | "components": "@/components", 15 | "utils": "@/lib/utils" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/components/back-button.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | import { Link } from './link'; 3 | import { usePathname } from 'next/navigation'; 4 | 5 | export const BackButton = () => { 6 | const pathname = usePathname(); 7 | if (pathname == '/') return null; 8 | return Back; 9 | }; 10 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/components/link.tsx: -------------------------------------------------------------------------------- 1 | import NextLink from 'next/link'; 2 | export const Link = ({ 3 | href, 4 | children, 5 | }: { 6 | href: string; 7 | children: React.ReactNode; 8 | }) => { 9 | return ( 10 | 11 | {children} 12 | 13 | ); 14 | }; 15 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/components/ui/input.tsx: -------------------------------------------------------------------------------- 1 | import * as React from 'react'; 2 | 3 | import { cn } from '@/lib/utils'; 4 | 5 | export interface InputProps 6 | extends React.InputHTMLAttributes {} 7 | 8 | const Input = React.forwardRef( 9 | ({ className, type, ...props }, ref) => { 10 | return ( 11 | 20 | ); 21 | } 22 | ); 23 | Input.displayName = 'Input'; 24 | 25 | export { Input }; 26 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/components/ui/label.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import * as React from 'react'; 4 | import * as LabelPrimitive from '@radix-ui/react-label'; 5 | import { cva, type VariantProps } from 'class-variance-authority'; 6 | 7 | import { cn } from '@/lib/utils'; 8 | 9 | const labelVariants = cva( 10 | 'text-sm font-medium leading-none peer-disabled:cursor-not-allowed peer-disabled:opacity-70' 11 | ); 12 | 13 | const Label = React.forwardRef< 14 | React.ElementRef, 15 | React.ComponentPropsWithoutRef & 16 | VariantProps 17 | >(({ className, ...props }, ref) => ( 18 | 23 | )); 24 | Label.displayName = LabelPrimitive.Root.displayName; 25 | 26 | export { Label }; 27 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/core/generate-text.ts: -------------------------------------------------------------------------------- 1 | import { openai } from '@ai-sdk/openai'; 2 | import { generateText } from 'ai'; 3 | import dotenv from 'dotenv'; 4 | 5 | dotenv.config(); 6 | 7 | async function main() { 8 | const result = await generateText({ 9 | model: openai('gpt-4o'), 10 | prompt: 'Tell me a joke.', 11 | }); 12 | 13 | console.log(result.text); 14 | } 15 | 16 | main().catch(console.error); 17 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/core/stream-text.ts: -------------------------------------------------------------------------------- 1 | import { openai } from '@ai-sdk/openai'; 2 | import { streamText } from 'ai'; 3 | import dotenv from 'dotenv'; 4 | 5 | dotenv.config(); 6 | 7 | async function main() { 8 | const result = await streamText({ 9 | model: openai('gpt-4o'), 10 | prompt: 'Tell me a joke.', 11 | }); 12 | 13 | for await (const textPart of result.textStream) { 14 | process.stdout.write(textPart); 15 | } 16 | } 17 | 18 | main().catch(console.error); 19 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/core/tools.ts: -------------------------------------------------------------------------------- 1 | import { openai } from '@ai-sdk/openai'; 2 | import { generateText, streamText } from 'ai'; 3 | import dotenv from 'dotenv'; 4 | import { z } from 'zod'; 5 | 6 | dotenv.config(); 7 | 8 | async function main() { 9 | const location = 'London'; 10 | const result = await generateText({ 11 | model: openai('gpt-4o'), 12 | prompt: `You are a funny chatbot. users location: ${location}`, 13 | tools: { 14 | weather: { 15 | description: "Get the weather for the user's location", 16 | parameters: z.object({ 17 | location: z.string().describe("user's location"), 18 | }), 19 | execute: async ({ location }) => { 20 | const temperature = Math.floor(Math.random() * 31); // call external api for {location} 21 | return { temperature }; 22 | }, 23 | }, 24 | }, 25 | }); 26 | 27 | if (result.toolResults && result.toolCalls) { 28 | const joke = await streamText({ 29 | model: openai('gpt-4o'), 30 | prompt: `Tell me a joke that incorporates ${location} 31 | and it's current temperature (${result.toolResults[0].result.temperature})`, 32 | }); 33 | 34 | for await (const textPart of joke.textStream) { 35 | process.stdout.write(textPart); 36 | } 37 | } 38 | } 39 | 40 | main().catch(console.error); 41 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/lib/utils.ts: -------------------------------------------------------------------------------- 1 | import { type ClassValue, clsx } from 'clsx'; 2 | import { twMerge } from 'tailwind-merge'; 3 | 4 | export function cn(...inputs: ClassValue[]) { 5 | return twMerge(clsx(inputs)); 6 | } 7 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/next-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | 4 | // NOTE: This file should not be edited 5 | // see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information. 6 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/next.config.mjs: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = {}; 3 | 4 | export default nextConfig; 5 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "one", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev --turbo", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@ai-sdk/openai": "^0.0.4", 13 | "@radix-ui/react-label": "^2.0.2", 14 | "@radix-ui/react-slot": "^1.0.2", 15 | "ai": "^3.3.26", 16 | "class-variance-authority": "^0.7.0", 17 | "clsx": "^2.1.1", 18 | "lucide-react": "^0.366.0", 19 | "nanoid": "^5.0.7", 20 | "next": "~14.2.3", 21 | "@portkey-ai/vercel-provider": "^1.0.1", 22 | "react": "^18.3.1", 23 | "react-dom": "^18.3.1", 24 | "server-only": "^0.0.1", 25 | "tailwind-merge": "^2.3.0", 26 | "tailwindcss-animate": "^1.0.7", 27 | "zod": "^3.23.8" 28 | }, 29 | "devDependencies": { 30 | "@types/node": "^20.12.12", 31 | "@types/react": "^18.3.2", 32 | "@types/react-dom": "^18.3.0", 33 | "autoprefixer": "^10.4.19", 34 | "dotenv": "^16.4.5", 35 | "eslint": "^8.57.0", 36 | "eslint-config-next": "14.1.4", 37 | "postcss": "^8.4.38", 38 | "tailwindcss": "^3.4.3", 39 | "tsx": "^4.10.3", 40 | "typescript": "^5.4.5" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cookbook/integrations/vercel/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["dom", "dom.iterable", "esnext"], 4 | "allowJs": true, 5 | "skipLibCheck": true, 6 | "strict": true, 7 | "noEmit": true, 8 | "esModuleInterop": true, 9 | "module": "esnext", 10 | "moduleResolution": "bundler", 11 | "resolveJsonModule": true, 12 | "isolatedModules": true, 13 | "jsx": "preserve", 14 | "incremental": true, 15 | "plugins": [ 16 | { 17 | "name": "next" 18 | } 19 | ], 20 | "paths": { 21 | "@/*": ["./*"] 22 | } 23 | }, 24 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], 25 | "exclude": ["node_modules"] 26 | } 27 | -------------------------------------------------------------------------------- /cookbook/use-cases/LMSYS Series/README.md: -------------------------------------------------------------------------------- 1 | # LMSYS Series 2 | 3 | LMSYS is THE defining leaderboard to compare raw capabilities of various LLMs. It has so far collected over **1,000,000** human pairwise comparisons to rank LLMs in Elo-scale. 4 | 5 | We are creating cookbooks that help you get the best value of of LMSYS leaderboard for **production use cases**, by leveraging Portkey's Gateway, Observability, and Prompt Management features. 6 | 7 | ## Cookbooks 8 | 9 | 1. [Comparing the Top 10 LMSYS models using a single function.](./comparing-top10-LMSYS-models-with-Portkey.ipynb) 10 | 2. RAG Benchmark Over LMSYS Models (coming soon!) -------------------------------------------------------------------------------- /deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: portkeyai 5 | --- 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | name: portkeyai 10 | namespace: portkeyai 11 | spec: 12 | replicas: 1 13 | revisionHistoryLimit: 3 14 | selector: 15 | matchLabels: 16 | app: portkeyai 17 | version: v1 18 | strategy: 19 | rollingUpdate: 20 | maxSurge: 25% 21 | maxUnavailable: 25% 22 | type: RollingUpdate 23 | template: 24 | metadata: 25 | labels: 26 | app: portkeyai 27 | version: v1 28 | spec: 29 | containers: 30 | - image: portkeyai/gateway 31 | imagePullPolicy: IfNotPresent 32 | name: portkeyai 33 | ports: 34 | - containerPort: 8787 35 | protocol: TCP 36 | resources: {} 37 | dnsPolicy: ClusterFirst 38 | restartPolicy: Always 39 | schedulerName: default-scheduler 40 | securityContext: {} 41 | --- 42 | apiVersion: v1 43 | kind: Service 44 | metadata: 45 | name: portkeyai 46 | namespace: portkeyai 47 | spec: 48 | ports: 49 | - port: 8787 50 | protocol: TCP 51 | targetPort: 8787 52 | selector: 53 | app: portkeyai 54 | version: v1 55 | sessionAffinity: None 56 | type: NodePort 57 | 58 | 59 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | web: 4 | ports: 5 | - "8787:8787" 6 | image: "portkeyai/gateway:latest" 7 | restart: always -------------------------------------------------------------------------------- /docs/images/Sticker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/Sticker.png -------------------------------------------------------------------------------- /docs/images/anthropic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/anthropic.png -------------------------------------------------------------------------------- /docs/images/anyscale.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/anyscale.png -------------------------------------------------------------------------------- /docs/images/azure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/azure.png -------------------------------------------------------------------------------- /docs/images/bard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/bard.png -------------------------------------------------------------------------------- /docs/images/cohere.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cohere.png -------------------------------------------------------------------------------- /docs/images/cookbook-header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbook-header.png -------------------------------------------------------------------------------- /docs/images/cookbooks/101-configs-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/101-configs-1.png -------------------------------------------------------------------------------- /docs/images/cookbooks/101-configs-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/101-configs-2.png -------------------------------------------------------------------------------- /docs/images/cookbooks/cache-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/cache-1.png -------------------------------------------------------------------------------- /docs/images/cookbooks/cache-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/cache-2.png -------------------------------------------------------------------------------- /docs/images/cookbooks/cache-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/cache-3.png -------------------------------------------------------------------------------- /docs/images/cookbooks/fallback-trace-id.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/fallback-trace-id.png -------------------------------------------------------------------------------- /docs/images/cookbooks/langchain-hub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/langchain-hub.png -------------------------------------------------------------------------------- /docs/images/cookbooks/logs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/logs.png -------------------------------------------------------------------------------- /docs/images/cookbooks/prompt-template.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/prompt-template.png -------------------------------------------------------------------------------- /docs/images/cookbooks/resilient-loadbalance-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/resilient-loadbalance-1.png -------------------------------------------------------------------------------- /docs/images/cookbooks/resilient-loadbalance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/resilient-loadbalance.png -------------------------------------------------------------------------------- /docs/images/cookbooks/supabase-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/supabase-1.png -------------------------------------------------------------------------------- /docs/images/cookbooks/supabase-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/supabase-2.png -------------------------------------------------------------------------------- /docs/images/cookbooks/supabase-3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/supabase-3.png -------------------------------------------------------------------------------- /docs/images/cookbooks/supabase-4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/supabase-4.png -------------------------------------------------------------------------------- /docs/images/cookbooks/supabase-5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/supabase-5.png -------------------------------------------------------------------------------- /docs/images/cookbooks/supabase-6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/supabase-6.png -------------------------------------------------------------------------------- /docs/images/cookbooks/supabase-7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/cookbooks/supabase-7.png -------------------------------------------------------------------------------- /docs/images/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/demo.gif -------------------------------------------------------------------------------- /docs/images/gateway-border.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/gateway-border.png -------------------------------------------------------------------------------- /docs/images/header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/header.png -------------------------------------------------------------------------------- /docs/images/header_new.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/header_new.png -------------------------------------------------------------------------------- /docs/images/localai.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/localai.png -------------------------------------------------------------------------------- /docs/images/openai.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/openai.png -------------------------------------------------------------------------------- /docs/images/palm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Portkey-AI/gateway/67efbb2627d3e4cff6bef738a851b75a87ed2ab9/docs/images/palm.png -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | import tseslint from 'typescript-eslint'; 2 | 3 | export default tseslint.config({ 4 | ignores: ['.wrangler/**', 'node_modules/**', 'build/**'], 5 | plugins: { 6 | '@typescript-eslint': tseslint.plugin, 7 | }, 8 | languageOptions: { 9 | parser: tseslint.parser, 10 | }, 11 | files: ['**/*.ts'], 12 | rules: { 13 | // '@typescript-eslint/no-unused-vars': 'error', 14 | 'no-duplicate-imports': 'error', 15 | '@typescript-eslint/no-namespace': 'off', 16 | }, 17 | }); 18 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest').JestConfigWithTsJest} **/ 2 | export default { 3 | testEnvironment: 'node', 4 | transform: { 5 | '^.+.tsx?$': ['ts-jest', {}], 6 | }, 7 | testTimeout: 30000, // Set default timeout to 30 seconds 8 | }; 9 | -------------------------------------------------------------------------------- /patches/@types+async-retry+1.4.5.patch: -------------------------------------------------------------------------------- 1 | diff --git a/node_modules/@types/async-retry/index.d.ts b/node_modules/@types/async-retry/index.d.ts 2 | index ed730fe..20c0f2c 100755 3 | --- a/node_modules/@types/async-retry/index.d.ts 4 | +++ b/node_modules/@types/async-retry/index.d.ts 5 | @@ -49,7 +49,7 @@ declare namespace AsyncRetry { 6 | * @param bail A function you can invoke to abort the retrying (bail). 7 | * @param attempt The attempt number. The absolute first attempt (before any retries) is `1`. 8 | */ 9 | - type RetryFunction = (bail: (e: Error) => void, attempt: number) => TRet | Promise; 10 | + type RetryFunction = (bail: (e: Error) => void, attempt: number, rateLimiter: any) => TRet | Promise; 11 | } 12 | 13 | export = AsyncRetry; 14 | -------------------------------------------------------------------------------- /patches/async-retry+1.3.3.patch: -------------------------------------------------------------------------------- 1 | diff --git a/node_modules/async-retry/lib/index.js b/node_modules/async-retry/lib/index.js 2 | index d8f0c8a..d7a545e 100644 3 | --- a/node_modules/async-retry/lib/index.js 4 | +++ b/node_modules/async-retry/lib/index.js 5 | @@ -39,7 +39,7 @@ function retry(fn, opts) { 6 | var val; 7 | 8 | try { 9 | - val = fn(bail, num); 10 | + val = fn(bail, num, op); 11 | } catch (err) { 12 | onError(err, num); 13 | return; 14 | -------------------------------------------------------------------------------- /plugins/aporia/aporia.test.ts: -------------------------------------------------------------------------------- 1 | import testCreds from './.creds.json'; 2 | import { handler as validateProjectHandler } from './validateProject'; 3 | 4 | function getParameters() { 5 | return { 6 | credentials: testCreds, 7 | projectID: testCreds.projectID, 8 | }; 9 | } 10 | 11 | describe('validateProject handler', () => { 12 | it('should fail if the apiKey is invalid', async () => { 13 | const eventType = 'beforeRequestHook'; 14 | const context = { 15 | request: { text: 'this is a test string for moderations' }, 16 | }; 17 | const parameters = JSON.parse(JSON.stringify(getParameters())); 18 | parameters.credentials.apiKey = 'invalid-api-key'; 19 | 20 | const result = await validateProjectHandler(context, parameters, eventType); 21 | console.log(result); 22 | expect(result).toBeDefined(); 23 | expect(result.verdict).toBe(false); 24 | expect(result.error).toBeDefined(); 25 | expect(result.data).toBeNull(); 26 | }); 27 | }); 28 | -------------------------------------------------------------------------------- /plugins/aporia/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "aporia", 3 | "description": "Aporia Guardrails mitigates LLM hallucinations, inappropriate responses, prompt injection attacks, and other unintended behaviors in real-time", 4 | "credentials": { 5 | "type": "object", 6 | "properties": { 7 | "apiKey": { 8 | "type": "string", 9 | "label": "API Key", 10 | "description": "Find your API key in the Aporia dashboard", 11 | "encrypted": true 12 | } 13 | }, 14 | "required": ["apiKey"] 15 | }, 16 | "functions": [ 17 | { 18 | "name": "Validate - Project", 19 | "id": "validateProject", 20 | "supportedHooks": ["beforeRequestHook", "afterRequestHook"], 21 | "type": "guardrail", 22 | "description": [ 23 | { 24 | "type": "subHeading", 25 | "text": "Check all the policies within an Aporia project" 26 | } 27 | ], 28 | "parameters": { 29 | "type": "object", 30 | "properties": { 31 | "projectID": { 32 | "type": "string", 33 | "label": "Project ID", 34 | "description": [ 35 | { 36 | "type": "subHeading", 37 | "text": "Enter the Aporia Project ID to validate" 38 | } 39 | ] 40 | } 41 | }, 42 | "required": ["projectID"] 43 | } 44 | } 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /plugins/azure/types.ts: -------------------------------------------------------------------------------- 1 | export interface AzureCredentials { 2 | resourceName: string; 3 | azureAuthMode: 'apiKey' | 'entra' | 'managed'; 4 | apiKey?: string; 5 | clientId?: string; 6 | clientSecret?: string; 7 | tenantId?: string; 8 | } 9 | -------------------------------------------------------------------------------- /plugins/build.ts: -------------------------------------------------------------------------------- 1 | import conf from '../conf.json'; 2 | import fs from 'fs'; 3 | 4 | const pluginsEnabled = conf.plugins_enabled; 5 | 6 | let importStrings: any = []; 7 | let funcStrings: any = {}; 8 | let funcs: any = {}; 9 | 10 | for (const plugin of pluginsEnabled) { 11 | const manifest = await import(`./${plugin}/manifest.json`); 12 | const functions = manifest.functions.map((func: any) => func.id); 13 | importStrings = [ 14 | ...importStrings, 15 | ...functions.map( 16 | (func: any) => 17 | `import { handler as ${manifest.id}${func} } from "./${plugin}/${func}"` 18 | ), 19 | ]; 20 | 21 | funcs[plugin] = {}; 22 | functions.forEach((func: any) => { 23 | funcs[plugin][func] = func; 24 | }); 25 | 26 | funcStrings[plugin] = []; 27 | for (let key in funcs[plugin]) { 28 | funcStrings[plugin].push(`"${key}": ${manifest.id}${funcs[plugin][key]}`); 29 | } 30 | } 31 | 32 | const indexFilePath = './plugins/index.ts'; 33 | 34 | let finalFuncStrings: any = []; 35 | for (let key in funcStrings) { 36 | finalFuncStrings.push( 37 | `\n "${key}": {\n ${funcStrings[key].join(',\n ')}\n }` 38 | ); 39 | } 40 | 41 | const content = `${importStrings.join('\n')}\n\nexport const plugins = {${finalFuncStrings}\n};\n`; 42 | 43 | fs.writeFileSync(indexFilePath, content); 44 | -------------------------------------------------------------------------------- /plugins/default/log.ts: -------------------------------------------------------------------------------- 1 | import { PluginContext, PluginHandler, PluginParameters } from '../types'; 2 | import { post } from '../utils'; 3 | 4 | export const handler: PluginHandler = async ( 5 | context: PluginContext, 6 | parameters: PluginParameters 7 | ) => { 8 | let error = null; 9 | let verdict = false; 10 | let data = null; 11 | 12 | try { 13 | let url = parameters.logURL; 14 | let headers: Record = parameters?.headers 15 | ? JSON.parse(parameters.headers) 16 | : {}; 17 | 18 | // log the request 19 | await post(url, context, { headers }, parameters.timeout || 3000); 20 | 21 | verdict = true; 22 | data = { message: `Logged the request to ${url}` }; 23 | } catch (e: any) { 24 | delete e.stack; 25 | error = e; 26 | } 27 | 28 | return { error, verdict, data }; 29 | }; 30 | -------------------------------------------------------------------------------- /plugins/pangea/version.ts: -------------------------------------------------------------------------------- 1 | export const VERSION = 'v1.0.0-beta'; 2 | -------------------------------------------------------------------------------- /plugins/panw-prisma-airs/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "panwPrismaAirs", 3 | "name": "PANW Prisma AIRS Guardrail", 4 | "description": "Blocks prompt/response when Palo Alto Networks Prisma AI Runtime Security returns action=block.", 5 | "credentials": [ 6 | { 7 | "id": "AIRS_API_KEY", 8 | "name": "AIRS API Key", 9 | "type": "string", 10 | "required": true 11 | } 12 | ], 13 | "functions": [ 14 | { 15 | "id": "intercept", 16 | "name": "PANW Prisma AIRS Guardrail", 17 | "type": "guardrail", 18 | "supportedHooks": ["beforeRequestHook", "afterRequestHook"], 19 | "parameters": { 20 | "type": "object", 21 | "properties": { 22 | "profile_name": { "type": "string" }, 23 | "ai_model": { "type": "string" }, 24 | "app_user": { "type": "string" } 25 | }, 26 | "required": ["profile_name"] 27 | } 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /plugins/panw-prisma-airs/panw.airs.test.ts: -------------------------------------------------------------------------------- 1 | import { handler as panwPrismaAirsHandler } from './intercept'; 2 | 3 | describe('PANW Prisma AIRS Guardrail', () => { 4 | const mockContext = { 5 | request: { text: 'This is a test prompt.' }, 6 | response: { text: 'This is a test response.' }, 7 | }; 8 | 9 | const params = { 10 | credentials: { AIRS_API_KEY: 'dummy-key' }, 11 | profile_name: 'test-profile', 12 | ai_model: 'gpt-unit-test', 13 | app_user: 'unit-tester', 14 | timeout: 3000, 15 | }; 16 | 17 | it('should return a result object with verdict, data, and error', async () => { 18 | const result = await panwPrismaAirsHandler( 19 | mockContext, 20 | params, 21 | 'beforeRequestHook' 22 | ); 23 | expect(result).toHaveProperty('verdict'); 24 | expect(result).toHaveProperty('data'); 25 | expect(result).toHaveProperty('error'); 26 | }); 27 | }); 28 | -------------------------------------------------------------------------------- /plugins/patronus/custom.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { postPatronus } from './globals'; 8 | 9 | export const handler: PluginHandler = async ( 10 | context: PluginContext, 11 | parameters: PluginParameters, 12 | eventType: HookEventType 13 | ) => { 14 | let error = null; 15 | let verdict = false; 16 | let data = null; 17 | 18 | const evaluator = 'judge'; 19 | const criteria = parameters.criteria; 20 | 21 | if (eventType !== 'afterRequestHook') { 22 | return { 23 | error: { 24 | message: 'Patronus guardrails only support after_request_hooks.', 25 | }, 26 | verdict: true, 27 | data, 28 | }; 29 | } 30 | 31 | const evaluationBody: any = { 32 | input: context.request.text, 33 | output: context.response.text, 34 | }; 35 | 36 | try { 37 | const result: any = await postPatronus( 38 | evaluator, 39 | parameters.credentials, 40 | evaluationBody, 41 | parameters.timeout || 15000, 42 | criteria 43 | ); 44 | 45 | const evalResult = result.results[0]; 46 | error = evalResult.error_message; 47 | 48 | // verdict can be true/false 49 | verdict = evalResult.evaluation_result.pass; 50 | 51 | data = evalResult.evaluation_result.additional_info; 52 | } catch (e: any) { 53 | delete e.stack; 54 | error = e; 55 | } 56 | 57 | return { error, verdict, data }; 58 | }; 59 | -------------------------------------------------------------------------------- /plugins/patronus/isConcise.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { postPatronus } from './globals'; 8 | 9 | export const handler: PluginHandler = async ( 10 | context: PluginContext, 11 | parameters: PluginParameters, 12 | eventType: HookEventType 13 | ) => { 14 | let error = null; 15 | let verdict = false; 16 | let data = null; 17 | 18 | const evaluator = 'judge'; 19 | const criteria = 'patronus:is-concise'; 20 | 21 | if (eventType !== 'afterRequestHook') { 22 | return { 23 | error: { 24 | message: 'Patronus guardrails only support after_request_hooks.', 25 | }, 26 | verdict: true, 27 | data, 28 | }; 29 | } 30 | 31 | const evaluationBody: any = { 32 | input: context.request.text, 33 | output: context.response.text, 34 | }; 35 | 36 | try { 37 | const result: any = await postPatronus( 38 | evaluator, 39 | parameters.credentials, 40 | evaluationBody, 41 | parameters.timeout || 15000, 42 | criteria 43 | ); 44 | 45 | const evalResult = result.results[0]; 46 | error = evalResult.error_message; 47 | 48 | // verdict can be true/false 49 | verdict = evalResult.evaluation_result.pass; 50 | 51 | data = evalResult.evaluation_result.additional_info; 52 | } catch (e: any) { 53 | delete e.stack; 54 | error = e; 55 | } 56 | 57 | return { error, verdict, data }; 58 | }; 59 | -------------------------------------------------------------------------------- /plugins/patronus/isHelpful.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { postPatronus } from './globals'; 8 | 9 | export const handler: PluginHandler = async ( 10 | context: PluginContext, 11 | parameters: PluginParameters, 12 | eventType: HookEventType 13 | ) => { 14 | let error = null; 15 | let verdict = false; 16 | let data = null; 17 | 18 | const evaluator = 'judge'; 19 | const criteria = 'patronus:is-helpful'; 20 | 21 | if (eventType !== 'afterRequestHook') { 22 | return { 23 | error: { 24 | message: 'Patronus guardrails only support after_request_hooks.', 25 | }, 26 | verdict: true, 27 | data, 28 | }; 29 | } 30 | 31 | const evaluationBody: any = { 32 | input: context.request.text, 33 | output: context.response.text, 34 | }; 35 | 36 | try { 37 | const result: any = await postPatronus( 38 | evaluator, 39 | parameters.credentials, 40 | evaluationBody, 41 | parameters.timeout || 15000, 42 | criteria 43 | ); 44 | 45 | const evalResult = result.results[0]; 46 | error = evalResult.error_message; 47 | 48 | // verdict can be true/false 49 | verdict = evalResult.evaluation_result.pass; 50 | 51 | data = evalResult.evaluation_result.additional_info; 52 | } catch (e: any) { 53 | delete e.stack; 54 | error = e; 55 | } 56 | 57 | return { error, verdict, data }; 58 | }; 59 | -------------------------------------------------------------------------------- /plugins/patronus/isPolite.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { postPatronus } from './globals'; 8 | 9 | export const handler: PluginHandler = async ( 10 | context: PluginContext, 11 | parameters: PluginParameters, 12 | eventType: HookEventType 13 | ) => { 14 | let error = null; 15 | let verdict = false; 16 | let data = null; 17 | 18 | const evaluator = 'judge'; 19 | const criteria = 'patronus:is-polite'; 20 | 21 | if (eventType !== 'afterRequestHook') { 22 | return { 23 | error: { 24 | message: 'Patronus guardrails only support after_request_hooks.', 25 | }, 26 | verdict: true, 27 | data, 28 | }; 29 | } 30 | 31 | const evaluationBody: any = { 32 | input: context.request.text, 33 | output: context.response.text, 34 | }; 35 | 36 | try { 37 | const result: any = await postPatronus( 38 | evaluator, 39 | parameters.credentials, 40 | evaluationBody, 41 | parameters.timeout || 15000, 42 | criteria 43 | ); 44 | 45 | const evalResult = result.results[0]; 46 | error = evalResult.error_message; 47 | 48 | // verdict can be true/false 49 | verdict = evalResult.evaluation_result.pass; 50 | 51 | data = evalResult.evaluation_result.additional_info; 52 | } catch (e: any) { 53 | delete e.stack; 54 | error = e; 55 | } 56 | 57 | return { error, verdict, data }; 58 | }; 59 | -------------------------------------------------------------------------------- /plugins/patronus/noApologies.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { postPatronus } from './globals'; 8 | 9 | export const handler: PluginHandler = async ( 10 | context: PluginContext, 11 | parameters: PluginParameters, 12 | eventType: HookEventType 13 | ) => { 14 | let error = null; 15 | let verdict = false; 16 | let data = null; 17 | 18 | const evaluator = 'judge'; 19 | const criteria = 'patronus:no-apologies'; 20 | 21 | if (eventType !== 'afterRequestHook') { 22 | return { 23 | error: { 24 | message: 'Patronus guardrails only support after_request_hooks.', 25 | }, 26 | verdict: true, 27 | data, 28 | }; 29 | } 30 | 31 | const evaluationBody: any = { 32 | input: context.request.text, 33 | output: context.response.text, 34 | }; 35 | 36 | try { 37 | const result: any = await postPatronus( 38 | evaluator, 39 | parameters.credentials, 40 | evaluationBody, 41 | parameters.timeout || 15000, 42 | criteria 43 | ); 44 | 45 | const evalResult = result.results[0]; 46 | error = evalResult.error_message; 47 | 48 | // verdict can be true/false 49 | verdict = evalResult.evaluation_result.pass; 50 | 51 | data = evalResult.evaluation_result.additional_info; 52 | } catch (e: any) { 53 | delete e.stack; 54 | error = e; 55 | } 56 | 57 | return { error, verdict, data }; 58 | }; 59 | -------------------------------------------------------------------------------- /plugins/patronus/noGenderBias.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { postPatronus } from './globals'; 8 | 9 | export const handler: PluginHandler = async ( 10 | context: PluginContext, 11 | parameters: PluginParameters, 12 | eventType: HookEventType 13 | ) => { 14 | let error = null; 15 | let verdict = false; 16 | let data = null; 17 | 18 | const evaluator = 'judge'; 19 | const criteria = 'patronus:no-gender-bias'; 20 | 21 | if (eventType !== 'afterRequestHook') { 22 | return { 23 | error: { 24 | message: 'Patronus guardrails only support after_request_hooks.', 25 | }, 26 | verdict: true, 27 | data, 28 | }; 29 | } 30 | 31 | const evaluationBody: any = { 32 | input: context.request.text, 33 | output: context.response.text, 34 | }; 35 | 36 | try { 37 | const result: any = await postPatronus( 38 | evaluator, 39 | parameters.credentials, 40 | evaluationBody, 41 | parameters.timeout || 15000, 42 | criteria 43 | ); 44 | 45 | const evalResult = result.results[0]; 46 | error = evalResult.error_message; 47 | 48 | // verdict can be true/false 49 | verdict = evalResult.evaluation_result.pass; 50 | 51 | data = evalResult.evaluation_result.additional_info; 52 | } catch (e: any) { 53 | delete e.stack; 54 | error = e; 55 | } 56 | 57 | return { error, verdict, data }; 58 | }; 59 | -------------------------------------------------------------------------------- /plugins/patronus/noRacialBias.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { postPatronus } from './globals'; 8 | 9 | export const handler: PluginHandler = async ( 10 | context: PluginContext, 11 | parameters: PluginParameters, 12 | eventType: HookEventType 13 | ) => { 14 | let error = null; 15 | let verdict = false; 16 | let data = null; 17 | 18 | const evaluator = 'judge'; 19 | const criteria = 'patronus:no-racial-bias'; 20 | 21 | if (eventType !== 'afterRequestHook') { 22 | return { 23 | error: { 24 | message: 'Patronus guardrails only support after_request_hooks.', 25 | }, 26 | verdict: true, 27 | data, 28 | }; 29 | } 30 | 31 | const evaluationBody: any = { 32 | input: context.request.text, 33 | output: context.response.text, 34 | }; 35 | 36 | try { 37 | const result: any = await postPatronus( 38 | evaluator, 39 | parameters.credentials, 40 | evaluationBody, 41 | parameters.timeout || 15000, 42 | criteria 43 | ); 44 | 45 | const evalResult = result.results[0]; 46 | error = evalResult.error_message; 47 | 48 | // verdict can be true/false 49 | verdict = evalResult.evaluation_result.pass; 50 | 51 | data = evalResult.evaluation_result.additional_info; 52 | } catch (e: any) { 53 | delete e.stack; 54 | error = e; 55 | } 56 | 57 | return { error, verdict, data }; 58 | }; 59 | -------------------------------------------------------------------------------- /plugins/patronus/retrievalAnswerRelevance.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { postPatronus } from './globals'; 8 | 9 | export const handler: PluginHandler = async ( 10 | context: PluginContext, 11 | parameters: PluginParameters, 12 | eventType: HookEventType 13 | ) => { 14 | let error = null; 15 | let verdict = false; 16 | let data = null; 17 | 18 | const evaluator = 'answer-relevance'; 19 | 20 | if (eventType !== 'afterRequestHook') { 21 | return { 22 | error: { 23 | message: 'Patronus guardrails only support after_request_hooks.', 24 | }, 25 | verdict: true, 26 | data, 27 | }; 28 | } 29 | 30 | const evaluationBody: any = { 31 | input: context.request.text, 32 | output: context.response.text, 33 | }; 34 | 35 | try { 36 | const result: any = await postPatronus( 37 | evaluator, 38 | parameters.credentials, 39 | evaluationBody, 40 | parameters.timeout || 15000 41 | ); 42 | 43 | const evalResult = result.results[0]; 44 | error = evalResult.error_message; 45 | 46 | // verdict can be true/false 47 | verdict = evalResult.evaluation_result.pass; 48 | 49 | data = evalResult.evaluation_result.additional_info; 50 | } catch (e: any) { 51 | delete e.stack; 52 | error = e; 53 | } 54 | 55 | return { error, verdict, data }; 56 | }; 57 | -------------------------------------------------------------------------------- /plugins/patronus/toxicity.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { postPatronus } from './globals'; 8 | 9 | export const handler: PluginHandler = async ( 10 | context: PluginContext, 11 | parameters: PluginParameters, 12 | eventType: HookEventType 13 | ) => { 14 | let error = null; 15 | let verdict = false; 16 | let data = null; 17 | 18 | const evaluator = 'toxicity'; 19 | 20 | if (eventType !== 'afterRequestHook') { 21 | return { 22 | error: { 23 | message: 'Patronus guardrails only support after_request_hooks.', 24 | }, 25 | verdict: true, 26 | data, 27 | }; 28 | } 29 | 30 | const evaluationBody: any = { 31 | input: context.request.text, 32 | output: context.response.text, 33 | }; 34 | 35 | try { 36 | const result: any = await postPatronus( 37 | evaluator, 38 | parameters.credentials, 39 | evaluationBody, 40 | parameters.timeout 41 | ); 42 | 43 | const evalResult = result.results[0]; 44 | error = evalResult.error_message; 45 | 46 | // verdict can be true/false 47 | verdict = evalResult.evaluation_result.pass; 48 | 49 | data = evalResult.evaluation_result.additional_info; 50 | } catch (e: any) { 51 | delete e.stack; 52 | error = e; 53 | } 54 | 55 | return { error, verdict, data }; 56 | }; 57 | -------------------------------------------------------------------------------- /plugins/pillar/globals.ts: -------------------------------------------------------------------------------- 1 | import { post } from '../utils'; 2 | 3 | export const PILLAR_BASE_URL = 'https://api.pillarseclabs.com/api/v1'; 4 | 5 | export const postPillar = async ( 6 | endpoint: string, 7 | credentials: any, 8 | data: any, 9 | timeout?: number 10 | ) => { 11 | const options = { 12 | headers: { 13 | Authorization: `Bearer ${credentials.apiKey}`, 14 | }, 15 | }; 16 | 17 | switch (endpoint) { 18 | case 'scanPrompt': 19 | return post(`${PILLAR_BASE_URL}/scan/prompt`, data, options, timeout); 20 | case 'scanResponse': 21 | return post(`${PILLAR_BASE_URL}/scan/response`, data, options, timeout); 22 | default: 23 | throw new Error(`Unknown Pillar endpoint: ${endpoint}`); 24 | } 25 | }; 26 | -------------------------------------------------------------------------------- /plugins/promptfoo/globals.ts: -------------------------------------------------------------------------------- 1 | import { post } from '../utils'; 2 | import { GuardResult, PIIResult, HarmResult, PromptfooResult } from './types'; 3 | 4 | export const PROMPTFOO_BASE_URL = 'https://api.promptfoo.dev/v1'; 5 | 6 | export const postPromptfoo = async < 7 | T extends GuardResult | PIIResult | HarmResult, 8 | >( 9 | endpoint: string, 10 | data: any, 11 | timeout?: number 12 | ): Promise> => { 13 | const options = { 14 | headers: { 15 | 'Content-Type': 'application/json', 16 | }, 17 | }; 18 | 19 | switch (endpoint) { 20 | case 'guard': 21 | return post( 22 | `${PROMPTFOO_BASE_URL}/guard`, 23 | data, 24 | options, 25 | timeout 26 | ) as Promise>; 27 | case 'pii': 28 | return post( 29 | `${PROMPTFOO_BASE_URL}/pii`, 30 | data, 31 | options, 32 | timeout 33 | ) as Promise>; 34 | case 'harm': 35 | return post( 36 | `${PROMPTFOO_BASE_URL}/harm`, 37 | data, 38 | options, 39 | timeout 40 | ) as Promise>; 41 | default: 42 | throw new Error(`Unknown Promptfoo endpoint: ${endpoint}`); 43 | } 44 | }; 45 | -------------------------------------------------------------------------------- /plugins/promptfoo/guard.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { getText } from '../utils'; 8 | import { postPromptfoo } from './globals'; 9 | import { GuardResult, PromptfooResult } from './types'; 10 | 11 | export const handler: PluginHandler = async ( 12 | context: PluginContext, 13 | parameters: PluginParameters, 14 | eventType: HookEventType 15 | ) => { 16 | let error = null; 17 | let verdict = true; 18 | let data = null; 19 | 20 | try { 21 | const guardObject = { 22 | input: getText(context, eventType), 23 | }; 24 | 25 | const result = await postPromptfoo( 26 | 'guard', 27 | guardObject, 28 | parameters.timeout 29 | ); 30 | 31 | // For now, we only check for jailbreak 32 | if (result.results[0].categories.jailbreak) { 33 | verdict = false; 34 | } 35 | 36 | data = result.results[0]; 37 | } catch (e: any) { 38 | delete e.stack; 39 | error = e; 40 | } 41 | 42 | return { error, verdict, data }; 43 | }; 44 | -------------------------------------------------------------------------------- /plugins/promptfoo/harm.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { getText } from '../utils'; 8 | import { postPromptfoo } from './globals'; 9 | import { HarmResult } from './types'; 10 | 11 | export const handler: PluginHandler = async ( 12 | context: PluginContext, 13 | parameters: PluginParameters, 14 | eventType: HookEventType 15 | ) => { 16 | let error = null; 17 | let verdict = true; 18 | let data = null; 19 | 20 | try { 21 | const harmObject = { 22 | input: getText(context, eventType), 23 | }; 24 | 25 | const result = await postPromptfoo( 26 | 'harm', 27 | harmObject, 28 | parameters.timeout 29 | ); 30 | 31 | // If any harm category is flagged, set verdict to false 32 | if (result.results[0].flagged) { 33 | verdict = false; 34 | } 35 | 36 | data = result.results[0]; 37 | } catch (e: any) { 38 | delete e.stack; 39 | error = e; 40 | } 41 | 42 | return { error, verdict, data }; 43 | }; 44 | -------------------------------------------------------------------------------- /plugins/promptsecurity/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "promptsecurity", 3 | "description": "Prompt Security Plugin", 4 | "credentials": { 5 | "type": "object", 6 | "properties": { 7 | "apiDomain": { 8 | "type": "string", 9 | "label": "API Domain", 10 | "description": "The domain of the Prompt Security API" 11 | }, 12 | "apiKey": { 13 | "type": "string", 14 | "label": "API Key", 15 | "description": "The API key for the Prompt Security API", 16 | "encrypted": true 17 | } 18 | }, 19 | "required": ["apiDomain", "apiKey"] 20 | }, 21 | "functions": [ 22 | { 23 | "name": "Protect Prompt", 24 | "id": "protectPrompt", 25 | "supportedHooks": ["beforeRequestHook"], 26 | "type": "guardrail", 27 | "description": [ 28 | { 29 | "type": "subHeading", 30 | "text": "Protect a user prompt before it is sent to the LLM." 31 | } 32 | ] 33 | }, 34 | { 35 | "name": "Protect Response", 36 | "id": "protectResponse", 37 | "supportedHooks": ["afterRequestHook"], 38 | "type": "guardrail", 39 | "description": [ 40 | { 41 | "type": "subHeading", 42 | "text": "Protect a LLM response before it is sent to the user." 43 | } 44 | ] 45 | } 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /plugins/promptsecurity/protectPrompt.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { getText } from '../utils'; 8 | import { promptSecurityProtectApi } from './shared'; 9 | 10 | export const handler: PluginHandler = async ( 11 | context: PluginContext, 12 | parameters: PluginParameters, 13 | eventType: HookEventType 14 | ) => { 15 | let error = null; 16 | let verdict = false; 17 | let data = null; 18 | try { 19 | let scanPromptObject: any = { prompt: getText(context, eventType) }; 20 | data = await promptSecurityProtectApi( 21 | parameters.credentials, 22 | scanPromptObject 23 | ); 24 | data = data.result.prompt; 25 | verdict = data.passed; 26 | } catch (e: any) { 27 | delete e.stack; 28 | error = e; 29 | } 30 | return { error, verdict, data }; 31 | }; 32 | -------------------------------------------------------------------------------- /plugins/promptsecurity/protectResponse.ts: -------------------------------------------------------------------------------- 1 | import { 2 | HookEventType, 3 | PluginContext, 4 | PluginHandler, 5 | PluginParameters, 6 | } from '../types'; 7 | import { getText } from '../utils'; 8 | import { promptSecurityProtectApi } from './shared'; 9 | 10 | export const handler: PluginHandler = async ( 11 | context: PluginContext, 12 | parameters: PluginParameters, 13 | eventType: HookEventType 14 | ) => { 15 | let error = null; 16 | let verdict = false; 17 | let data = null; 18 | try { 19 | let scanResponseObject: any = { response: getText(context, eventType) }; 20 | data = await promptSecurityProtectApi( 21 | parameters.credentials, 22 | scanResponseObject 23 | ); 24 | data = data.result.response; 25 | verdict = data.passed; 26 | } catch (e: any) { 27 | delete e.stack; 28 | error = e; 29 | } 30 | return { error, verdict, data }; 31 | }; 32 | -------------------------------------------------------------------------------- /plugins/promptsecurity/shared.ts: -------------------------------------------------------------------------------- 1 | import { post } from '../utils'; 2 | 3 | export const promptSecurityProtectApi = async (credentials: any, data: any) => { 4 | const headers = { 5 | 'APP-ID': credentials.apiKey, 6 | 'Content-Type': 'application/json', 7 | }; 8 | const url = `https://${credentials.apiDomain}/api/protect`; 9 | return post(url, data, { headers }); 10 | }; 11 | -------------------------------------------------------------------------------- /plugins/types.ts: -------------------------------------------------------------------------------- 1 | export interface PluginContext { 2 | [key: string]: any; 3 | requestType?: 'complete' | 'chatComplete' | 'embed'; 4 | provider?: string; 5 | } 6 | 7 | export interface PluginParameters> { 8 | [key: string]: any; 9 | credentials?: K; 10 | } 11 | 12 | export interface PluginHandlerResponse { 13 | error: any; 14 | verdict?: boolean; 15 | // The data object can be any JSON object or null. 16 | data?: any | null; 17 | transformedData?: any; 18 | transformed?: boolean; 19 | } 20 | 21 | export type HookEventType = 'beforeRequestHook' | 'afterRequestHook'; 22 | 23 | export type PluginHandler

> = ( 24 | context: PluginContext, 25 | parameters: PluginParameters

, 26 | eventType: HookEventType, 27 | options?: { 28 | env: Record; 29 | getFromCacheByKey?: (key: string) => Promise; 30 | putInCacheWithValue?: (key: string, value: any) => Promise; 31 | } 32 | ) => Promise; 33 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import typescript from '@rollup/plugin-typescript'; 2 | import terser from '@rollup/plugin-terser'; 3 | import json from '@rollup/plugin-json'; 4 | import copy from 'rollup-plugin-copy'; 5 | 6 | export default { 7 | input: 'src/start-server.ts', 8 | output: { 9 | dir: 'build', 10 | format: 'es', 11 | }, 12 | plugins: [ 13 | typescript({ 14 | exclude: ['**/*.test.ts', 'start-test.js', 'cookbook', 'docs'], 15 | }), 16 | terser(), 17 | json(), 18 | copy({ 19 | targets: [{ src: 'src/public/*', dest: 'build/public' }], 20 | }), 21 | ], 22 | }; 23 | -------------------------------------------------------------------------------- /src/errors/GatewayError.ts: -------------------------------------------------------------------------------- 1 | export class GatewayError extends Error { 2 | constructor( 3 | message: string, 4 | public cause?: Error 5 | ) { 6 | super(message); 7 | this.name = 'GatewayError'; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/errors/RouterError.ts: -------------------------------------------------------------------------------- 1 | export class RouterError extends Error { 2 | constructor( 3 | message: string, 4 | public cause?: Error 5 | ) { 6 | super(message); 7 | this.name = 'RouterError'; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/handlers/batchesHandler.ts: -------------------------------------------------------------------------------- 1 | import { Context } from 'hono'; 2 | import { 3 | constructConfigFromRequestHeaders, 4 | tryTargetsRecursively, 5 | } from './handlerUtils'; 6 | import { endpointStrings } from '../providers/types'; 7 | 8 | function batchesHandler(endpoint: endpointStrings, method: 'POST' | 'GET') { 9 | async function handler(c: Context): Promise { 10 | try { 11 | let requestHeaders = Object.fromEntries(c.req.raw.headers); 12 | let request = endpoint === 'createBatch' ? await c.req.json() : {}; 13 | const camelCaseConfig = constructConfigFromRequestHeaders(requestHeaders); 14 | const tryTargetsResponse = await tryTargetsRecursively( 15 | c, 16 | camelCaseConfig ?? {}, 17 | request, 18 | requestHeaders, 19 | endpoint, 20 | method, 21 | 'config' 22 | ); 23 | 24 | return tryTargetsResponse; 25 | } catch (err: any) { 26 | console.error({ message: `${endpoint} error ${err.message}` }); 27 | return new Response( 28 | JSON.stringify({ 29 | status: 'failure', 30 | message: 'Something went wrong', 31 | }), 32 | { 33 | status: 500, 34 | headers: { 35 | 'content-type': 'application/json', 36 | }, 37 | } 38 | ); 39 | } 40 | } 41 | return handler; 42 | } 43 | 44 | export default batchesHandler; 45 | -------------------------------------------------------------------------------- /src/handlers/filesHandler.ts: -------------------------------------------------------------------------------- 1 | import { Context } from 'hono'; 2 | import { 3 | constructConfigFromRequestHeaders, 4 | tryTargetsRecursively, 5 | } from './handlerUtils'; 6 | import { endpointStrings } from '../providers/types'; 7 | 8 | function filesHandler( 9 | endpoint: endpointStrings, 10 | method: 'POST' | 'GET' | 'DELETE' 11 | ) { 12 | async function handler(c: Context): Promise { 13 | try { 14 | const requestHeaders = Object.fromEntries(c.req.raw.headers); 15 | const camelCaseConfig = constructConfigFromRequestHeaders(requestHeaders); 16 | let body = {}; 17 | if (c.req.raw.body instanceof ReadableStream) { 18 | body = c.req.raw.body; 19 | } 20 | const tryTargetsResponse = await tryTargetsRecursively( 21 | c, 22 | camelCaseConfig ?? {}, 23 | body, 24 | requestHeaders, 25 | endpoint, 26 | method, 27 | 'config' 28 | ); 29 | 30 | return tryTargetsResponse; 31 | } catch (err: any) { 32 | console.error({ message: `${endpoint} error ${err.message}` }); 33 | return new Response( 34 | JSON.stringify({ 35 | status: 'failure', 36 | message: 'Something went wrong', 37 | }), 38 | { 39 | status: 500, 40 | headers: { 41 | 'content-type': 'application/json', 42 | }, 43 | } 44 | ); 45 | } 46 | } 47 | return handler; 48 | } 49 | 50 | export default filesHandler; 51 | -------------------------------------------------------------------------------- /src/handlers/modelResponsesHandler.ts: -------------------------------------------------------------------------------- 1 | import { Context } from 'hono'; 2 | import { 3 | constructConfigFromRequestHeaders, 4 | tryTargetsRecursively, 5 | } from './handlerUtils'; 6 | import { endpointStrings } from '../providers/types'; 7 | 8 | function modelResponsesHandler( 9 | endpoint: endpointStrings, 10 | method: 'POST' | 'GET' | 'DELETE' 11 | ) { 12 | async function handler(c: Context): Promise { 13 | try { 14 | let requestHeaders = Object.fromEntries(c.req.raw.headers); 15 | let request = method === 'POST' ? await c.req.json() : {}; 16 | const camelCaseConfig = constructConfigFromRequestHeaders(requestHeaders); 17 | const tryTargetsResponse = await tryTargetsRecursively( 18 | c, 19 | camelCaseConfig ?? {}, 20 | request, 21 | requestHeaders, 22 | endpoint, 23 | method, 24 | 'config' 25 | ); 26 | 27 | return tryTargetsResponse; 28 | } catch (err: any) { 29 | console.error({ message: `${endpoint} error ${err.message}` }); 30 | return new Response( 31 | JSON.stringify({ 32 | status: 'failure', 33 | message: 'Something went wrong', 34 | }), 35 | { 36 | status: 500, 37 | headers: { 38 | 'content-type': 'application/json', 39 | }, 40 | } 41 | ); 42 | } 43 | } 44 | return handler; 45 | } 46 | 47 | export default modelResponsesHandler; 48 | -------------------------------------------------------------------------------- /src/handlers/modelsHandler.ts: -------------------------------------------------------------------------------- 1 | import { Context } from 'hono'; 2 | import models from '../data/models.json'; 3 | import providers from '../data/providers.json'; 4 | 5 | /** 6 | * Handles the models request. Returns a list of models supported by the Ai gateway. 7 | * Allows filters in query params for the provider 8 | * @param c - The Hono context 9 | * @returns - The response 10 | */ 11 | export async function modelsHandler(c: Context): Promise { 12 | // If the request does not contain a provider query param, return all models. Add a count as well. 13 | const provider = c.req.query('provider'); 14 | if (!provider) { 15 | return c.json({ 16 | ...models, 17 | count: models.data.length, 18 | }); 19 | } else { 20 | // Filter the models by the provider 21 | const filteredModels = models.data.filter( 22 | (model: any) => model.provider.id === provider 23 | ); 24 | return c.json({ 25 | ...models, 26 | data: filteredModels, 27 | count: filteredModels.length, 28 | }); 29 | } 30 | } 31 | 32 | export async function providersHandler(c: Context): Promise { 33 | return c.json({ 34 | ...providers, 35 | count: providers.data.length, 36 | }); 37 | } 38 | -------------------------------------------------------------------------------- /src/middlewares/hooks/globals.ts: -------------------------------------------------------------------------------- 1 | export const HOOKS_EVENT_TYPE_PRESETS = { 2 | SYNC_BEFORE_REQUEST_HOOK: 'syncBeforeRequestHook', 3 | ASYNC_BEFORE_REQUEST_HOOK: 'asyncBeforeRequestHook', 4 | SYNC_AFTER_REQUEST_HOOK: 'syncAfterRequestHook', 5 | ASYNC_AFTER_REQUEST_HOOK: 'asyncAfterRequestHook', 6 | }; 7 | -------------------------------------------------------------------------------- /src/providers/ai21/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const AI21APIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.ai21.com/studio/v1', 5 | headers: ({ providerOptions }) => { 6 | const headers: Record = { 7 | Authorization: `Bearer ${providerOptions.apiKey}`, 8 | }; 9 | return headers; 10 | }, 11 | getEndpoint: ({ fn, gatewayRequestBodyJSON }) => { 12 | const { model } = gatewayRequestBodyJSON; 13 | switch (fn) { 14 | case 'complete': { 15 | return `/${model}/complete`; 16 | } 17 | case 'chatComplete': { 18 | return `/${model}/chat`; 19 | } 20 | case 'embed': { 21 | return `/embed`; 22 | } 23 | default: 24 | return ''; 25 | } 26 | }, 27 | }; 28 | 29 | export default AI21APIConfig; 30 | -------------------------------------------------------------------------------- /src/providers/ai21/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import AI21APIConfig from './api'; 3 | import { 4 | AI21ChatCompleteConfig, 5 | AI21ChatCompleteResponseTransform, 6 | } from './chatComplete'; 7 | import { AI21CompleteConfig, AI21CompleteResponseTransform } from './complete'; 8 | import { AI21EmbedConfig, AI21EmbedResponseTransform } from './embed'; 9 | 10 | const AI21Config: ProviderConfigs = { 11 | complete: AI21CompleteConfig, 12 | chatComplete: AI21ChatCompleteConfig, 13 | embed: AI21EmbedConfig, 14 | api: AI21APIConfig, 15 | responseTransforms: { 16 | complete: AI21CompleteResponseTransform, 17 | chatComplete: AI21ChatCompleteResponseTransform, 18 | embed: AI21EmbedResponseTransform, 19 | }, 20 | }; 21 | 22 | export default AI21Config; 23 | -------------------------------------------------------------------------------- /src/providers/anthropic/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const AnthropicAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.anthropic.com/v1', 5 | headers: ({ providerOptions, fn, gatewayRequestBody }) => { 6 | const headers: Record = { 7 | 'X-API-Key': `${providerOptions.apiKey}`, 8 | }; 9 | 10 | // Accept anthropic_beta and anthropic_version in body to support enviroments which cannot send it in headers. 11 | const betaHeader = 12 | providerOptions?.['anthropicBeta'] ?? 13 | gatewayRequestBody?.['anthropic_beta'] ?? 14 | 'messages-2023-12-15'; 15 | const version = 16 | providerOptions?.['anthropicVersion'] ?? 17 | gatewayRequestBody?.['anthropic_version'] ?? 18 | '2023-06-01'; 19 | 20 | if (fn === 'chatComplete') { 21 | headers['anthropic-beta'] = betaHeader; 22 | } 23 | headers['anthropic-version'] = version; 24 | return headers; 25 | }, 26 | getEndpoint: ({ fn }) => { 27 | switch (fn) { 28 | case 'complete': 29 | return '/complete'; 30 | case 'chatComplete': 31 | return '/messages'; 32 | default: 33 | return ''; 34 | } 35 | }, 36 | }; 37 | 38 | export default AnthropicAPIConfig; 39 | -------------------------------------------------------------------------------- /src/providers/anthropic/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import AnthropicAPIConfig from './api'; 3 | import { 4 | AnthropicChatCompleteConfig, 5 | AnthropicChatCompleteResponseTransform, 6 | AnthropicChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | import { 9 | AnthropicCompleteConfig, 10 | AnthropicCompleteResponseTransform, 11 | AnthropicCompleteStreamChunkTransform, 12 | } from './complete'; 13 | 14 | const AnthropicConfig: ProviderConfigs = { 15 | complete: AnthropicCompleteConfig, 16 | chatComplete: AnthropicChatCompleteConfig, 17 | api: AnthropicAPIConfig, 18 | responseTransforms: { 19 | 'stream-complete': AnthropicCompleteStreamChunkTransform, 20 | complete: AnthropicCompleteResponseTransform, 21 | chatComplete: AnthropicChatCompleteResponseTransform, 22 | 'stream-chatComplete': AnthropicChatCompleteStreamChunkTransform, 23 | }, 24 | }; 25 | 26 | export default AnthropicConfig; 27 | -------------------------------------------------------------------------------- /src/providers/anthropic/types.ts: -------------------------------------------------------------------------------- 1 | export type AnthropicStreamState = { 2 | toolIndex?: number; 3 | usage?: { 4 | prompt_tokens?: number; 5 | completion_tokens?: number; 6 | cache_read_input_tokens?: number; 7 | cache_creation_input_tokens?: number; 8 | }; 9 | model?: string; 10 | }; 11 | -------------------------------------------------------------------------------- /src/providers/anyscale/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const AnyscaleAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.endpoints.anyscale.com/v1', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/chat/completions'; 12 | case 'complete': 13 | return '/completions'; 14 | case 'embed': 15 | return '/embeddings'; 16 | default: 17 | return ''; 18 | } 19 | }, 20 | }; 21 | 22 | export default AnyscaleAPIConfig; 23 | -------------------------------------------------------------------------------- /src/providers/anyscale/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import AnyscaleAPIConfig from './api'; 3 | import { 4 | AnyscaleChatCompleteConfig, 5 | AnyscaleChatCompleteResponseTransform, 6 | AnyscaleChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | import { 9 | AnyscaleCompleteConfig, 10 | AnyscaleCompleteResponseTransform, 11 | AnyscaleCompleteStreamChunkTransform, 12 | } from './complete'; 13 | import { AnyscaleEmbedConfig, AnyscaleEmbedResponseTransform } from './embed'; 14 | 15 | const AnyscaleConfig: ProviderConfigs = { 16 | complete: AnyscaleCompleteConfig, 17 | chatComplete: AnyscaleChatCompleteConfig, 18 | embed: AnyscaleEmbedConfig, 19 | api: AnyscaleAPIConfig, 20 | responseTransforms: { 21 | 'stream-complete': AnyscaleCompleteStreamChunkTransform, 22 | complete: AnyscaleCompleteResponseTransform, 23 | chatComplete: AnyscaleChatCompleteResponseTransform, 24 | 'stream-chatComplete': AnyscaleChatCompleteStreamChunkTransform, 25 | embed: AnyscaleEmbedResponseTransform, 26 | }, 27 | }; 28 | 29 | export default AnyscaleConfig; 30 | -------------------------------------------------------------------------------- /src/providers/azure-ai-inference/embed.ts: -------------------------------------------------------------------------------- 1 | import { EmbedResponse } from '../../types/embedRequestBody'; 2 | import { OpenAIErrorResponseTransform } from '../openai/utils'; 3 | import { ErrorResponse, ProviderConfig } from '../types'; 4 | 5 | export const AzureAIInferenceEmbedConfig: ProviderConfig = { 6 | model: { 7 | param: 'model', 8 | required: false, 9 | }, 10 | input: { 11 | param: 'input', 12 | required: true, 13 | }, 14 | user: { 15 | param: 'user', 16 | }, 17 | }; 18 | 19 | interface AzureAIInferenceEmbedResponse extends EmbedResponse {} 20 | 21 | export const AzureAIInferenceEmbedResponseTransform = (provider: string) => { 22 | const transformer: ( 23 | response: AzureAIInferenceEmbedResponse | ErrorResponse, 24 | responseStatus: number 25 | ) => EmbedResponse | ErrorResponse = (response, responseStatus) => { 26 | if (responseStatus !== 200 && 'error' in response) { 27 | return OpenAIErrorResponseTransform(response, provider); 28 | } 29 | 30 | return { ...response, provider: provider }; 31 | }; 32 | 33 | return transformer; 34 | }; 35 | -------------------------------------------------------------------------------- /src/providers/azure-openai/createBatch.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfig } from '../types'; 2 | 3 | export const AzureOpenAICreateBatchConfig: ProviderConfig = { 4 | input_file_id: { 5 | param: 'input_file_id', 6 | required: true, 7 | }, 8 | endpoint: { 9 | param: 'endpoint', 10 | required: true, 11 | }, 12 | completion_window: { 13 | param: 'completion_window', 14 | default: '24h', 15 | required: true, 16 | }, 17 | metadata: { 18 | param: 'metadata', 19 | required: false, 20 | }, 21 | }; 22 | -------------------------------------------------------------------------------- /src/providers/azure-openai/createFinetune.ts: -------------------------------------------------------------------------------- 1 | import { FinetuneRequest } from '../types'; 2 | 3 | export const AzureTransformFinetuneBody = (body: FinetuneRequest) => { 4 | const _body = { ...body }; 5 | 6 | if (_body.method && !_body.hyperparameters) { 7 | const hyperparameters = 8 | _body.method[_body.method.type]?.hyperparameters ?? {}; 9 | _body.hyperparameters = { ...hyperparameters } as any; 10 | 11 | delete _body.method; 12 | } 13 | 14 | return { 15 | ..._body, 16 | }; 17 | }; 18 | -------------------------------------------------------------------------------- /src/providers/azure-openai/createSpeech.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse, ProviderConfig } from '../types'; 3 | import { OpenAIErrorResponseTransform } from '../openai/utils'; 4 | 5 | export const AzureOpenAICreateSpeechConfig: ProviderConfig = { 6 | model: { 7 | param: 'model', 8 | required: true, 9 | default: 'tts-1', 10 | }, 11 | input: { 12 | param: 'input', 13 | required: true, 14 | }, 15 | voice: { 16 | param: 'voice', 17 | required: true, 18 | default: 'alloy', 19 | }, 20 | response_format: { 21 | param: 'response_format', 22 | required: false, 23 | default: 'mp3', 24 | }, 25 | speed: { 26 | param: 'speed', 27 | required: false, 28 | default: 1, 29 | }, 30 | }; 31 | 32 | export const AzureOpenAICreateSpeechResponseTransform: ( 33 | response: Response | ErrorResponse, 34 | responseStatus: number 35 | ) => Response | ErrorResponse = (response, responseStatus) => { 36 | if (responseStatus !== 200 && 'error' in response) { 37 | return OpenAIErrorResponseTransform(response, OPEN_AI); 38 | } 39 | 40 | return response; 41 | }; 42 | -------------------------------------------------------------------------------- /src/providers/azure-openai/createTranscription.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from '../openai/utils'; 4 | 5 | export const AzureOpenAICreateTranscriptionResponseTransform: ( 6 | response: Response | ErrorResponse, 7 | responseStatus: number 8 | ) => Response | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/azure-openai/createTranslation.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from '../openai/utils'; 4 | 5 | export const AzureOpenAICreateTranslationResponseTransform: ( 6 | response: Response | ErrorResponse, 7 | responseStatus: number 8 | ) => Response | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/azure-openai/uploadFile.ts: -------------------------------------------------------------------------------- 1 | export const AzureOpenAIRequestTransform = (requestBody: ReadableStream) => { 2 | return requestBody; 3 | }; 4 | -------------------------------------------------------------------------------- /src/providers/bedrock/cancelBatch.ts: -------------------------------------------------------------------------------- 1 | import { CancelBatchResponse, ErrorResponse } from '../types'; 2 | import { BedrockErrorResponseTransform } from './chatComplete'; 3 | import { BedrockErrorResponse } from './embed'; 4 | 5 | export const BedrockCancelBatchResponseTransform = ( 6 | response: Response | BedrockErrorResponse, 7 | responseStatus: number, 8 | _responseHeaders: Headers, 9 | _strictOpenAiCompliance: boolean, 10 | gatewayRequestUrl: string 11 | ): CancelBatchResponse | ErrorResponse => { 12 | if (responseStatus !== 200) { 13 | const errorResponse = BedrockErrorResponseTransform( 14 | response as BedrockErrorResponse 15 | ); 16 | if (errorResponse) return errorResponse; 17 | } 18 | const batchId = decodeURIComponent( 19 | gatewayRequestUrl.split('/v1/batches/')[1].split('/')[0] 20 | ); 21 | 22 | return { 23 | status: 'success', 24 | object: 'batch', 25 | id: batchId, 26 | }; 27 | }; 28 | -------------------------------------------------------------------------------- /src/providers/bedrock/deleteFile.ts: -------------------------------------------------------------------------------- 1 | import { GatewayError } from '../../errors/GatewayError'; 2 | import { ErrorResponse } from '../types'; 3 | 4 | export const BedrockDeleteFileResponseTransform = (): 5 | | Response 6 | | ErrorResponse => { 7 | throw new GatewayError(`deleteFile is not supported by Bedrock`); 8 | }; 9 | -------------------------------------------------------------------------------- /src/providers/bedrock/listFinetunes.ts: -------------------------------------------------------------------------------- 1 | import { ErrorResponse } from '../types'; 2 | import { BedrockErrorResponseTransform } from './chatComplete'; 3 | import { BedrockFinetuneRecord } from './types'; 4 | import { bedrockFinetuneToOpenAI } from './utils'; 5 | 6 | export const BedrockListFinetuneResponseTransform: ( 7 | response: any | ErrorResponse, 8 | responseStatus: number 9 | ) => Response | ErrorResponse = (response, responseStatus) => { 10 | if (responseStatus !== 200) { 11 | return BedrockErrorResponseTransform(response) || response; 12 | } 13 | const records = 14 | response?.modelCustomizationJobSummaries as BedrockFinetuneRecord[]; 15 | const openaiRecords = records.map(bedrockFinetuneToOpenAI); 16 | return { 17 | data: openaiRecords, 18 | object: 'list', 19 | total_count: openaiRecords.length, 20 | last_id: response?.nextToken, 21 | }; 22 | }; 23 | -------------------------------------------------------------------------------- /src/providers/bedrock/listfiles.ts: -------------------------------------------------------------------------------- 1 | import { GatewayError } from '../../errors/GatewayError'; 2 | import { ErrorResponse } from '../types'; 3 | 4 | export const BedrockListFilesResponseTransform = (): 5 | | Response 6 | | ErrorResponse => { 7 | throw new GatewayError(`listFiles is not supported by Bedrock`); 8 | }; 9 | -------------------------------------------------------------------------------- /src/providers/bedrock/retrieveFinetune.ts: -------------------------------------------------------------------------------- 1 | import { ErrorResponse } from '../types'; 2 | import { BedrockErrorResponseTransform } from './chatComplete'; 3 | import { bedrockFinetuneToOpenAI } from './utils'; 4 | 5 | export const BedrockFinetuneResponseTransform: ( 6 | response: Response | ErrorResponse, 7 | responseStatus: number 8 | ) => Response | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200) { 10 | return BedrockErrorResponseTransform(response as any) || response; 11 | } 12 | 13 | return bedrockFinetuneToOpenAI(response as any) as any; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/cerebras/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | export const cerebrasAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.cerebras.ai/v1', 5 | headers: ({ providerOptions }) => { 6 | return { 7 | Authorization: `Bearer ${providerOptions.apiKey}`, 8 | 'User-Agent': 'Portkey Gateway/1.0', 9 | }; 10 | }, 11 | getEndpoint: ({ fn }) => { 12 | switch (fn) { 13 | case 'chatComplete': 14 | return '/chat/completions'; 15 | default: 16 | return ''; 17 | } 18 | }, 19 | }; 20 | -------------------------------------------------------------------------------- /src/providers/cerebras/index.ts: -------------------------------------------------------------------------------- 1 | import { CEREBRAS } from '../../globals'; 2 | import { chatCompleteParams, responseTransformers } from '../open-ai-base'; 3 | import { ProviderConfigs } from '../types'; 4 | import { cerebrasAPIConfig } from './api'; 5 | 6 | export const cerebrasProviderAPIConfig: ProviderConfigs = { 7 | chatComplete: chatCompleteParams([ 8 | 'frequency_penalty', 9 | 'logit_bias', 10 | 'logprobs', 11 | 'presence_penalty', 12 | 'parallel_tool_calls', 13 | 'service_tier', 14 | ]), 15 | api: cerebrasAPIConfig, 16 | responseTransforms: responseTransformers(CEREBRAS, { 17 | chatComplete: true, 18 | }), 19 | }; 20 | -------------------------------------------------------------------------------- /src/providers/cohere/cancelBatch.ts: -------------------------------------------------------------------------------- 1 | import { CancelBatchResponse, ErrorResponse } from '../types'; 2 | import { CohereErrorResponse } from './types'; 3 | import { CohereErrorResponseTransform } from './utils'; 4 | 5 | export const CohereCancelBatchResponseTransform = ( 6 | response: Response | CohereErrorResponse, 7 | responseStatus: number, 8 | _responseHeaders: Record, 9 | _strictOpenAiCompliance: boolean, 10 | gatewayRequestUrl: string 11 | ): CancelBatchResponse | ErrorResponse => { 12 | if (responseStatus !== 200 && 'message' in response) { 13 | return CohereErrorResponseTransform(response); 14 | } 15 | return { 16 | status: 'success', 17 | object: 'batch', 18 | id: 19 | gatewayRequestUrl?.split('/v1/batches/')?.[1]?.replace('/cancel', '') || 20 | '', 21 | }; 22 | }; 23 | -------------------------------------------------------------------------------- /src/providers/cohere/deleteFile.ts: -------------------------------------------------------------------------------- 1 | import { DeleteFileResponse, ErrorResponse } from '../types'; 2 | import { CohereErrorResponse } from './types'; 3 | import { CohereErrorResponseTransform } from './utils'; 4 | 5 | export const CohereDeleteFileResponseTransform: ( 6 | response: Response | CohereErrorResponse, 7 | responseStatus: number, 8 | _responseHeaders: Record, 9 | _strictOpenAiCompliance: boolean, 10 | gatewayRequestUrl: string 11 | ) => DeleteFileResponse | ErrorResponse = ( 12 | response, 13 | responseStatus, 14 | _responseHeaders, 15 | _strictOpenAiCompliance, 16 | gatewayRequestUrl 17 | ) => { 18 | if (responseStatus !== 200 && 'message' in response) { 19 | return CohereErrorResponseTransform(response); 20 | } 21 | const id = gatewayRequestUrl.split('/').pop() || ''; 22 | return { 23 | object: 'file', 24 | deleted: true, 25 | id, 26 | }; 27 | }; 28 | -------------------------------------------------------------------------------- /src/providers/cohere/listBatches.ts: -------------------------------------------------------------------------------- 1 | import { COHERE } from '../../globals'; 2 | import { ErrorResponse, ListBatchesResponse } from '../types'; 3 | import { generateInvalidProviderResponseError } from '../utils'; 4 | import { CohereErrorResponse, CohereListBatchResponse } from './types'; 5 | import { CohereErrorResponseTransform } from './utils'; 6 | 7 | export const CohereListBatchResponseTransform: ( 8 | response: CohereListBatchResponse | CohereErrorResponse, 9 | responseStatus: number, 10 | _responseHeaders: Headers, 11 | strictOpenAiCompliance: boolean 12 | ) => ListBatchesResponse | ErrorResponse = ( 13 | response, 14 | responseStatus, 15 | _responseHeaders, 16 | strictOpenAiCompliance 17 | ) => { 18 | if (responseStatus !== 200 && 'message' in response) { 19 | return CohereErrorResponseTransform(response); 20 | } 21 | 22 | if ('embed_jobs' in response) { 23 | return { 24 | object: 'list', 25 | data: response.embed_jobs.map((job) => ({ 26 | id: job.job_id, 27 | object: 'batch', 28 | created_at: new Date(job.created_at).getTime(), 29 | status: job.status, 30 | input_file_id: job.input_dataset_id, 31 | output_file_id: job.output_dataset_id, 32 | metadata: job.meta, 33 | ...(strictOpenAiCompliance 34 | ? { model: job.model, truncate: job.truncate, name: job.name } 35 | : {}), 36 | })), 37 | }; 38 | } 39 | 40 | return generateInvalidProviderResponseError(response, COHERE); 41 | }; 42 | -------------------------------------------------------------------------------- /src/providers/cohere/utils.ts: -------------------------------------------------------------------------------- 1 | import { COHERE } from '../../globals'; 2 | import { generateErrorResponse } from '../utils'; 3 | import { CohereErrorResponse } from './types'; 4 | 5 | export const CohereErrorResponseTransform = (response: CohereErrorResponse) => { 6 | return generateErrorResponse( 7 | { 8 | message: response.message, 9 | type: null, 10 | param: null, 11 | code: null, 12 | }, 13 | COHERE 14 | ); 15 | }; 16 | -------------------------------------------------------------------------------- /src/providers/cortex/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const CortexAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: ({ providerOptions }) => 5 | `https://${(providerOptions as any).snowflakeAccount}.snowflakecomputing.com/api/v2`, 6 | headers: ({ providerOptions }) => ({ 7 | 'X-Snowflake-Authorization-Token-Type': 'KEYPAIR_JWT', 8 | Authorization: `Bearer ${providerOptions.apiKey}`, 9 | 'Content-Type': 'application/json', 10 | Accept: 'application/json, text/event-stream', 11 | }), 12 | getEndpoint: ({ fn }) => { 13 | switch (fn) { 14 | case 'chatComplete': 15 | return '/cortex/inference:complete'; 16 | default: 17 | return ''; 18 | } 19 | }, 20 | }; 21 | 22 | export default CortexAPIConfig; 23 | -------------------------------------------------------------------------------- /src/providers/cortex/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import CortexAPIConfig from './api'; 3 | import { CORTEX } from '../../globals'; 4 | import { 5 | chatCompleteParams, 6 | completeParams, 7 | embedParams, 8 | responseTransformers, 9 | } from '../open-ai-base'; 10 | 11 | const CortexConfig: ProviderConfigs = { 12 | chatComplete: chatCompleteParams([], { model: 'mistral-large' }), 13 | complete: completeParams([], { model: 'mistral-large' }), 14 | embed: embedParams([], { model: 'mistral-large' }), 15 | api: CortexAPIConfig, 16 | responseTransforms: responseTransformers(CORTEX, { 17 | chatComplete: true, 18 | }), 19 | }; 20 | 21 | export default CortexConfig; 22 | -------------------------------------------------------------------------------- /src/providers/dashscope/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | export const dashscopeAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://dashscope.aliyuncs.com/compatible-mode/v1', 5 | headers({ providerOptions }) { 6 | const { apiKey } = providerOptions; 7 | return { Authorization: `Bearer ${apiKey}` }; 8 | }, 9 | getEndpoint({ fn }) { 10 | switch (fn) { 11 | case 'chatComplete': 12 | return `/chat/completions`; 13 | case 'embed': 14 | return `/embeddings`; 15 | default: 16 | return ''; 17 | } 18 | }, 19 | }; 20 | -------------------------------------------------------------------------------- /src/providers/dashscope/index.ts: -------------------------------------------------------------------------------- 1 | import { DASHSCOPE } from '../../globals'; 2 | import { 3 | chatCompleteParams, 4 | embedParams, 5 | responseTransformers, 6 | } from '../open-ai-base'; 7 | import { ProviderConfigs } from '../types'; 8 | import { dashscopeAPIConfig } from './api'; 9 | 10 | export const DashScopeConfig: ProviderConfigs = { 11 | chatComplete: chatCompleteParams([], { model: 'qwen-turbo' }), 12 | embed: embedParams([], { model: 'text-embedding-v1' }), 13 | api: dashscopeAPIConfig, 14 | responseTransforms: responseTransformers(DASHSCOPE, { 15 | chatComplete: true, 16 | embed: true, 17 | }), 18 | }; 19 | -------------------------------------------------------------------------------- /src/providers/deepbricks/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const DeepbricksAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.deepbricks.ai/v1', 5 | headers: ({ providerOptions }) => { 6 | const headersObj: Record = { 7 | Authorization: `Bearer ${providerOptions.apiKey}`, 8 | }; 9 | if (providerOptions.openaiOrganization) { 10 | headersObj['Deepbricks-Organization'] = 11 | providerOptions.openaiOrganization; 12 | } 13 | 14 | if (providerOptions.openaiProject) { 15 | headersObj['Deepbricks-Project'] = providerOptions.openaiProject; 16 | } 17 | 18 | return headersObj; 19 | }, 20 | getEndpoint: ({ fn }) => { 21 | switch (fn) { 22 | case 'chatComplete': 23 | return '/chat/completions'; 24 | case 'imageGenerate': 25 | return '/images/generations'; 26 | default: 27 | return ''; 28 | } 29 | }, 30 | }; 31 | 32 | export default DeepbricksAPIConfig; 33 | -------------------------------------------------------------------------------- /src/providers/deepbricks/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import DeepbricksAPIConfig from './api'; 3 | import { 4 | DeepbricksChatCompleteConfig, 5 | DeepbricksChatCompleteResponseTransform, 6 | } from './chatComplete'; 7 | import { 8 | DeepbricksImageGenerateConfig, 9 | DeepbricksImageGenerateResponseTransform, 10 | } from './imageGenerate'; 11 | 12 | const DeepbricksConfig: ProviderConfigs = { 13 | api: DeepbricksAPIConfig, 14 | chatComplete: DeepbricksChatCompleteConfig, 15 | imageGenerate: DeepbricksImageGenerateConfig, 16 | responseTransforms: { 17 | chatComplete: DeepbricksChatCompleteResponseTransform, 18 | imageGenerate: DeepbricksImageGenerateResponseTransform, 19 | }, 20 | }; 21 | 22 | export default DeepbricksConfig; 23 | -------------------------------------------------------------------------------- /src/providers/deepinfra/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const DeepInfraApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.deepinfra.com/v1/openai', 5 | headers: ({ providerOptions }) => { 6 | return { 7 | Authorization: `Bearer ${providerOptions.apiKey}`, 8 | }; 9 | }, 10 | getEndpoint: ({ fn }) => { 11 | switch (fn) { 12 | case 'chatComplete': 13 | return '/chat/completions'; 14 | default: 15 | return ''; 16 | } 17 | }, 18 | }; 19 | 20 | export default DeepInfraApiConfig; 21 | -------------------------------------------------------------------------------- /src/providers/deepinfra/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import DeepInfraApiConfig from './api'; 3 | import { 4 | DeepInfraChatCompleteConfig, 5 | DeepInfraChatCompleteResponseTransform, 6 | DeepInfraChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | 9 | const DeepInfraConfig: ProviderConfigs = { 10 | chatComplete: DeepInfraChatCompleteConfig, 11 | api: DeepInfraApiConfig, 12 | responseTransforms: { 13 | chatComplete: DeepInfraChatCompleteResponseTransform, 14 | 'stream-chatComplete': DeepInfraChatCompleteStreamChunkTransform, 15 | }, 16 | }; 17 | 18 | export default DeepInfraConfig; 19 | -------------------------------------------------------------------------------- /src/providers/deepseek/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const DeepSeekAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.deepseek.com', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; // https://platform.deepseek.com/api_keys 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/v1/chat/completions'; 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default DeepSeekAPIConfig; 19 | -------------------------------------------------------------------------------- /src/providers/deepseek/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import DeepSeekAPIConfig from './api'; 3 | import { 4 | DeepSeekChatCompleteConfig, 5 | DeepSeekChatCompleteResponseTransform, 6 | DeepSeekChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | 9 | const DeepSeekConfig: ProviderConfigs = { 10 | chatComplete: DeepSeekChatCompleteConfig, 11 | api: DeepSeekAPIConfig, 12 | responseTransforms: { 13 | chatComplete: DeepSeekChatCompleteResponseTransform, 14 | 'stream-chatComplete': DeepSeekChatCompleteStreamChunkTransform, 15 | }, 16 | }; 17 | 18 | export default DeepSeekConfig; 19 | -------------------------------------------------------------------------------- /src/providers/fireworks-ai/listFiles.ts: -------------------------------------------------------------------------------- 1 | import { 2 | FireworksAIErrorResponse, 3 | FireworksAIErrorResponseTransform, 4 | FireworksAIValidationErrorResponse, 5 | } from './chatComplete'; 6 | import { FireworksFile } from './types'; 7 | import { fireworksDatasetToOpenAIFile } from './utils'; 8 | 9 | export const FireworksFileListResponseTransform = ( 10 | response: (FireworksAIValidationErrorResponse | FireworksAIErrorResponse) & { 11 | datasets: FireworksFile[]; 12 | totalSize: number; 13 | }, 14 | responseStatus: number 15 | ) => { 16 | if (responseStatus === 200) { 17 | const datasets = response.datasets as FireworksFile[]; 18 | const records = datasets.map(fireworksDatasetToOpenAIFile); 19 | return { 20 | object: 'list', 21 | data: records, 22 | last_id: records.at(-1)?.id, 23 | has_more: response.totalSize > response.datasets.length, 24 | total: response.totalSize, 25 | }; 26 | } 27 | 28 | return FireworksAIErrorResponseTransform(response); 29 | }; 30 | -------------------------------------------------------------------------------- /src/providers/fireworks-ai/retrieveFile.ts: -------------------------------------------------------------------------------- 1 | import { FireworksFile } from './types'; 2 | import { fireworksDatasetToOpenAIFile } from './utils'; 3 | 4 | export const FireworksFileRetrieveResponseTransform = ( 5 | response: FireworksFile, 6 | responseStatus: number 7 | ) => { 8 | if (responseStatus === 200) { 9 | return fireworksDatasetToOpenAIFile(response); 10 | } 11 | return response; 12 | }; 13 | -------------------------------------------------------------------------------- /src/providers/fireworks-ai/types.ts: -------------------------------------------------------------------------------- 1 | export interface FireworksFile { 2 | createTime: string; 3 | displayName: string; 4 | exampleCount: number; 5 | format: 'UNSPECIFIED_FORMAT' | 'CHAT' | 'COMPLETION'; 6 | name: string; 7 | state: 'UPLOADING' | 'READY' | 'UNSPECIFIED'; 8 | status: { 9 | code: 10 | | 'OK' 11 | | 'CANCELLED' 12 | | 'UNKNOWN' 13 | | 'INVALID_ARGUMENT' 14 | | 'DEADLINE_EXCEEDED' 15 | | 'NOT_FOUND' 16 | | 'ALREADY_EXISTS' 17 | | 'PERMISSION_DENIED' 18 | | 'UNAUTHENTICATED' 19 | | 'RESOURCE_EXHAUSTED' 20 | | 'FAILED_PRECONDITION' 21 | | 'ABORTED' 22 | | 'OUT_OF_RANGE' 23 | | 'UNIMPLEMENTED' 24 | | 'INTERNAL' 25 | | 'UNAVAILABLE' 26 | | 'DATA_LOSS'; 27 | message: string; 28 | }; 29 | userUploaded: Record; 30 | } 31 | -------------------------------------------------------------------------------- /src/providers/fireworks-ai/uploadFile.ts: -------------------------------------------------------------------------------- 1 | export const FireworksFileUploadResponseTransform = ( 2 | response: Response, 3 | responseStatus: number 4 | ) => { 5 | return response; 6 | }; 7 | -------------------------------------------------------------------------------- /src/providers/fireworks-ai/utils.ts: -------------------------------------------------------------------------------- 1 | import { FireworksFile } from './types'; 2 | 3 | export const fireworksDatasetToOpenAIFile = (dataset: FireworksFile) => { 4 | const name = dataset.displayName || dataset.name; 5 | const id = name.split('/').at(-1); 6 | return { 7 | id: id, 8 | filename: `${id}.jsonl`, 9 | // Doesn't support batches, so default to fine-tune 10 | purpose: 'fine-tune', 11 | organisation_id: name.split('/').at(1), 12 | status: dataset.state.toLowerCase(), 13 | created_at: dataset.createTime, 14 | object: 'file', 15 | }; 16 | }; 17 | -------------------------------------------------------------------------------- /src/providers/google-vertex-ai/cancelBatch.ts: -------------------------------------------------------------------------------- 1 | export const GoogleCancelBatchResponseTransform = (response: Response) => { 2 | return response; 3 | }; 4 | -------------------------------------------------------------------------------- /src/providers/google-vertex-ai/createFinetune.ts: -------------------------------------------------------------------------------- 1 | import { GOOGLE_VERTEX_AI } from '../../globals'; 2 | import { ProviderConfig } from '../types'; 3 | import { GoogleErrorResponse, GoogleFinetuneRecord } from './types'; 4 | import { GoogleToOpenAIFinetune, transformVertexFinetune } from './utils'; 5 | 6 | export const GoogleVertexFinetuneConfig: ProviderConfig = { 7 | model: { 8 | param: 'baseModel', 9 | required: true, 10 | }, 11 | training_file: { 12 | param: 'supervisedTuningSpec', 13 | required: true, 14 | transform: transformVertexFinetune, 15 | }, 16 | suffix: { 17 | param: 'tunedModelDisplayName', 18 | required: true, 19 | }, 20 | validation_file: { 21 | param: 'supervisedTuningSpec', 22 | required: false, 23 | transform: transformVertexFinetune, 24 | }, 25 | method: { 26 | param: 'supervisedTuningSpec', 27 | required: false, 28 | transform: transformVertexFinetune, 29 | }, 30 | hyperparameters: { 31 | param: 'supervisedTuningSpec', 32 | required: false, 33 | transform: transformVertexFinetune, 34 | }, 35 | }; 36 | 37 | export const GoogleFinetuneCreateResponseTransform = ( 38 | input: Response | GoogleErrorResponse, 39 | status: number 40 | ) => { 41 | if (status !== 200) { 42 | return { ...input, provider: GOOGLE_VERTEX_AI }; 43 | } 44 | return GoogleToOpenAIFinetune(input as unknown as GoogleFinetuneRecord); 45 | }; 46 | -------------------------------------------------------------------------------- /src/providers/google-vertex-ai/listBatches.ts: -------------------------------------------------------------------------------- 1 | import { GOOGLE_VERTEX_AI } from '../../globals'; 2 | import { generateInvalidProviderResponseError } from '../utils'; 3 | import { GoogleBatchRecord, GoogleErrorResponse } from './types'; 4 | import { GoogleToOpenAIBatch } from './utils'; 5 | 6 | type GoogleListBatchesResponse = { 7 | batchPredictionJobs: GoogleBatchRecord[]; 8 | nextPageToken?: string; 9 | }; 10 | 11 | export const GoogleListBatchesResponseTransform = ( 12 | response: GoogleListBatchesResponse | GoogleErrorResponse, 13 | responseStatus: number 14 | ) => { 15 | if (responseStatus !== 200) { 16 | return generateInvalidProviderResponseError(response, GOOGLE_VERTEX_AI); 17 | } 18 | 19 | const batches = 20 | (response as { batchPredictionJobs: GoogleBatchRecord[] }) 21 | .batchPredictionJobs ?? []; 22 | 23 | const objects = batches.map(GoogleToOpenAIBatch); 24 | 25 | return { 26 | data: objects, 27 | object: 'list', 28 | first_id: objects.at(0)?.id, 29 | last_id: objects.at(-1)?.id, 30 | has_more: !!(response as GoogleListBatchesResponse)?.nextPageToken, 31 | }; 32 | }; 33 | -------------------------------------------------------------------------------- /src/providers/google-vertex-ai/listFiles.ts: -------------------------------------------------------------------------------- 1 | import { GOOGLE_VERTEX_AI } from '../../globals'; 2 | 3 | export const GoogleListFilesRequestHandler = async () => { 4 | return new Response( 5 | JSON.stringify({ 6 | message: 'listFiles is not supported by Google Vertex AI', 7 | status: 'failure', 8 | provider: GOOGLE_VERTEX_AI, 9 | }), 10 | { status: 500, headers: { 'Content-Type': 'application/json' } } 11 | ); 12 | }; 13 | -------------------------------------------------------------------------------- /src/providers/google-vertex-ai/listFinetunes.ts: -------------------------------------------------------------------------------- 1 | import { GOOGLE_VERTEX_AI } from '../../globals'; 2 | import { GoogleFinetuneRecord, GoogleErrorResponse } from './types'; 3 | import { GoogleToOpenAIFinetune } from './utils'; 4 | 5 | type GoogleListFinetunesResponse = { 6 | tuningJobs: GoogleFinetuneRecord[]; 7 | nextPageToken?: string; 8 | }; 9 | 10 | export const GoogleFinetuneListResponseTransform = ( 11 | input: Response | GoogleErrorResponse, 12 | status: number 13 | ) => { 14 | if (status !== 200) { 15 | return { ...input, provider: GOOGLE_VERTEX_AI }; 16 | } 17 | const records = 18 | (input as unknown as { tuningJobs: GoogleFinetuneRecord[] }).tuningJobs ?? 19 | []; 20 | 21 | const objects = records.map(GoogleToOpenAIFinetune); 22 | 23 | return { 24 | data: objects, 25 | object: 'list', 26 | first_id: objects.at(0)?.id, 27 | last_id: objects.at(-1)?.id, 28 | has_more: !!(input as unknown as GoogleListFinetunesResponse) 29 | ?.nextPageToken, 30 | }; 31 | }; 32 | -------------------------------------------------------------------------------- /src/providers/google-vertex-ai/retrieveBatch.ts: -------------------------------------------------------------------------------- 1 | // GoogleToOpenAIBatch 2 | 3 | import { GoogleBatchRecord } from './types'; 4 | import { GoogleToOpenAIBatch } from './utils'; 5 | 6 | export const GoogleRetrieveBatchResponseTransform = ( 7 | response: GoogleBatchRecord, 8 | status: number 9 | ) => { 10 | if (status !== 200) { 11 | return response; 12 | } 13 | 14 | return GoogleToOpenAIBatch(response); 15 | }; 16 | -------------------------------------------------------------------------------- /src/providers/google-vertex-ai/retrieveFileContent.ts: -------------------------------------------------------------------------------- 1 | import { GoogleErrorResponseTransform } from './utils'; 2 | 3 | export const GoogleRetrieveFileContentResponseTransform = ( 4 | response: Response, 5 | status: number 6 | ) => { 7 | if (status !== 200) { 8 | return GoogleErrorResponseTransform(response as any) || response; 9 | } 10 | return response; 11 | }; 12 | -------------------------------------------------------------------------------- /src/providers/google-vertex-ai/retrieveFinetune.ts: -------------------------------------------------------------------------------- 1 | import { GoogleFinetuneRecord } from './types'; 2 | import { GoogleToOpenAIFinetune } from './utils'; 3 | 4 | export const GoogleFinetuneRetrieveResponseTransform = ( 5 | response: Response, 6 | status: number 7 | ) => { 8 | if (status !== 200) { 9 | return response; 10 | } 11 | 12 | return GoogleToOpenAIFinetune(response as unknown as GoogleFinetuneRecord); 13 | }; 14 | -------------------------------------------------------------------------------- /src/providers/google/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | export const GoogleApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://generativelanguage.googleapis.com', 5 | headers: () => { 6 | return { 'Content-Type': 'application/json' }; 7 | }, 8 | getEndpoint: ({ fn, providerOptions, gatewayRequestBodyJSON }) => { 9 | let routeVersion = 'v1beta'; 10 | let mappedFn = fn; 11 | const { model, stream } = gatewayRequestBodyJSON; 12 | if (model?.includes('gemini-2.0-flash-thinking-exp')) { 13 | routeVersion = 'v1alpha'; 14 | } 15 | const { apiKey } = providerOptions; 16 | if (stream && fn === 'chatComplete') { 17 | return `/${routeVersion}/models/${model}:streamGenerateContent?key=${apiKey}`; 18 | } 19 | switch (mappedFn) { 20 | case 'chatComplete': { 21 | return `/${routeVersion}/models/${model}:generateContent?key=${apiKey}`; 22 | } 23 | case 'embed': { 24 | return `/${routeVersion}/models/${model}:embedContent?key=${apiKey}`; 25 | } 26 | default: 27 | return ''; 28 | } 29 | }, 30 | }; 31 | 32 | export default GoogleApiConfig; 33 | -------------------------------------------------------------------------------- /src/providers/google/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import GoogleApiConfig from './api'; 3 | import { 4 | GoogleChatCompleteConfig, 5 | GoogleChatCompleteResponseTransform, 6 | GoogleChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | import { GoogleEmbedConfig, GoogleEmbedResponseTransform } from './embed'; 9 | 10 | const GoogleConfig: ProviderConfigs = { 11 | api: GoogleApiConfig, 12 | chatComplete: GoogleChatCompleteConfig, 13 | embed: GoogleEmbedConfig, 14 | responseTransforms: { 15 | chatComplete: GoogleChatCompleteResponseTransform, 16 | 'stream-chatComplete': GoogleChatCompleteStreamChunkTransform, 17 | embed: GoogleEmbedResponseTransform, 18 | }, 19 | }; 20 | 21 | export default GoogleConfig; 22 | -------------------------------------------------------------------------------- /src/providers/groq/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const GroqAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.groq.com/openai/v1', 5 | headers: ({ providerOptions, fn }) => { 6 | const headersObj: Record = { 7 | Authorization: `Bearer ${providerOptions.apiKey}`, 8 | }; 9 | if (fn === 'createTranscription' || fn === 'createTranslation') 10 | headersObj['Content-Type'] = 'multipart/form-data'; 11 | return headersObj; 12 | }, 13 | getEndpoint: ({ fn }) => { 14 | switch (fn) { 15 | case 'chatComplete': 16 | return '/chat/completions'; 17 | case 'createTranscription': 18 | return '/audio/transcriptions'; 19 | case 'createTranslation': 20 | return '/audio/translations'; 21 | case 'createSpeech': 22 | return '/audio/speech'; 23 | case 'createModelResponse': 24 | return '/chat/completions'; 25 | default: 26 | return ''; 27 | } 28 | }, 29 | }; 30 | 31 | export default GroqAPIConfig; 32 | -------------------------------------------------------------------------------- /src/providers/groq/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import GroqAPIConfig from './api'; 3 | import { GroqChatCompleteStreamChunkTransform } from './chatComplete'; 4 | import { 5 | chatCompleteParams, 6 | responseTransformers, 7 | createSpeechParams, 8 | } from '../open-ai-base'; 9 | import { GROQ } from '../../globals'; 10 | 11 | const GroqConfig: ProviderConfigs = { 12 | api: GroqAPIConfig, 13 | chatComplete: chatCompleteParams(['logprobs', 'logits_bias', 'top_logprobs']), 14 | createTranscription: {}, 15 | createTranslation: {}, 16 | createSpeech: createSpeechParams([]), 17 | responseTransforms: { 18 | ...responseTransformers(GROQ, { 19 | chatComplete: true, 20 | createSpeech: true, 21 | }), 22 | 'stream-chatComplete': GroqChatCompleteStreamChunkTransform, 23 | }, 24 | }; 25 | 26 | export default GroqConfig; 27 | -------------------------------------------------------------------------------- /src/providers/huggingface/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const HuggingfaceAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: ({ providerOptions }) => { 5 | return ( 6 | providerOptions.huggingfaceBaseUrl || 7 | 'https://api-inference.huggingface.co' 8 | ); 9 | }, 10 | headers: ({ providerOptions }) => ({ 11 | Authorization: `Bearer ${providerOptions.apiKey}`, 12 | }), 13 | getEndpoint: ({ fn, gatewayRequestBodyJSON, providerOptions }) => { 14 | const { model } = gatewayRequestBodyJSON; 15 | const modelPath = providerOptions.huggingfaceBaseUrl 16 | ? '' 17 | : `/models/${model}`; 18 | switch (fn) { 19 | case 'chatComplete': 20 | return `${modelPath}/v1/chat/completions`; 21 | case 'complete': 22 | return `${modelPath}/v1/completions`; 23 | default: 24 | return ''; 25 | } 26 | }, 27 | }; 28 | 29 | export default HuggingfaceAPIConfig; 30 | -------------------------------------------------------------------------------- /src/providers/huggingface/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import { 3 | HuggingfaceCompleteConfig, 4 | HuggingfaceCompleteResponseTransform, 5 | HuggingfaceCompleteStreamChunkTransform, 6 | } from './complete'; 7 | import HuggingfaceAPIConfig from './api'; 8 | import { 9 | HuggingfaceChatCompleteConfig, 10 | HuggingfaceChatCompleteResponseTransform, 11 | HuggingfaceChatCompleteStreamChunkTransform, 12 | } from './chatComplete'; 13 | 14 | const HuggingfaceConfig: ProviderConfigs = { 15 | complete: HuggingfaceCompleteConfig, 16 | api: HuggingfaceAPIConfig, 17 | chatComplete: HuggingfaceChatCompleteConfig, 18 | responseTransforms: { 19 | complete: HuggingfaceCompleteResponseTransform, 20 | 'stream-complete': HuggingfaceCompleteStreamChunkTransform, 21 | chatComplete: HuggingfaceChatCompleteResponseTransform, 22 | 'stream-chatComplete': HuggingfaceChatCompleteStreamChunkTransform, 23 | }, 24 | }; 25 | 26 | export default HuggingfaceConfig; 27 | -------------------------------------------------------------------------------- /src/providers/huggingface/types.ts: -------------------------------------------------------------------------------- 1 | export interface HuggingfaceErrorResponse { 2 | error: string; 3 | } 4 | -------------------------------------------------------------------------------- /src/providers/huggingface/utils.ts: -------------------------------------------------------------------------------- 1 | import { HUGGING_FACE } from '../../globals'; 2 | import { ErrorResponse } from '../types'; 3 | import { generateErrorResponse } from '../utils'; 4 | import { HuggingfaceErrorResponse } from './types'; 5 | 6 | export const HuggingfaceErrorResponseTransform: ( 7 | response: HuggingfaceErrorResponse, 8 | responseStatus: number 9 | ) => ErrorResponse = (response, responseStatus) => { 10 | return generateErrorResponse( 11 | { 12 | message: response.error, 13 | type: null, 14 | param: null, 15 | code: responseStatus.toString(), 16 | }, 17 | HUGGING_FACE 18 | ); 19 | }; 20 | -------------------------------------------------------------------------------- /src/providers/inference-net/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | export const inferenceAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.inference.net/v1', 5 | headers({ providerOptions }) { 6 | const { apiKey } = providerOptions; 7 | return { Authorization: `Bearer ${apiKey}` }; 8 | }, 9 | getEndpoint({ fn }) { 10 | switch (fn) { 11 | case 'chatComplete': 12 | return `/chat/completions`; 13 | default: 14 | return ''; 15 | } 16 | }, 17 | }; 18 | -------------------------------------------------------------------------------- /src/providers/inference-net/index.ts: -------------------------------------------------------------------------------- 1 | import { INFERENCENET } from '../../globals'; 2 | import { chatCompleteParams, responseTransformers } from '../open-ai-base'; 3 | import { ProviderConfigs } from '../types'; 4 | import { inferenceAPIConfig } from './api'; 5 | 6 | export const InferenceNetProviderConfigs: ProviderConfigs = { 7 | chatComplete: chatCompleteParams([], { model: 'llama3' }), 8 | api: inferenceAPIConfig, 9 | responseTransforms: responseTransformers(INFERENCENET, { 10 | chatComplete: true, 11 | }), 12 | }; 13 | -------------------------------------------------------------------------------- /src/providers/jina/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const JinaAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.jina.ai/v1', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'embed': 11 | return '/embeddings'; 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default JinaAPIConfig; 19 | -------------------------------------------------------------------------------- /src/providers/jina/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import JinaAPIConfig from './api'; 3 | import { JinaEmbedConfig, JinaEmbedResponseTransform } from './embed'; 4 | 5 | const JinaConfig: ProviderConfigs = { 6 | embed: JinaEmbedConfig, 7 | api: JinaAPIConfig, 8 | responseTransforms: { 9 | embed: JinaEmbedResponseTransform, 10 | }, 11 | }; 12 | 13 | export default JinaConfig; 14 | -------------------------------------------------------------------------------- /src/providers/lambda/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | export const LambdaAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => `https://api.lambdalabs.com/v1`, 5 | headers({ providerOptions }) { 6 | const { apiKey } = providerOptions; 7 | const headers = { 8 | Authorization: `Bearer ${apiKey}`, 9 | }; 10 | return headers; 11 | }, 12 | getEndpoint({ fn }) { 13 | switch (fn) { 14 | case 'chatComplete': { 15 | return '/chat/completions'; 16 | } 17 | case 'complete': { 18 | return '/completions'; 19 | } 20 | default: 21 | return ''; 22 | } 23 | }, 24 | }; 25 | -------------------------------------------------------------------------------- /src/providers/lambda/index.ts: -------------------------------------------------------------------------------- 1 | import { LAMBDA } from '../../globals'; 2 | import { 3 | chatCompleteParams, 4 | completeParams, 5 | responseTransformers, 6 | } from '../open-ai-base'; 7 | import { ProviderConfigs } from '../types'; 8 | import { LambdaAPIConfig } from './api'; 9 | 10 | export const LambdaProviderConfig: ProviderConfigs = { 11 | chatComplete: chatCompleteParams([], { model: 'Liquid-AI-40B' }), 12 | complete: completeParams([], { model: 'Liquid-AI-40B' }), 13 | api: LambdaAPIConfig, 14 | responseTransforms: responseTransformers(LAMBDA, { 15 | chatComplete: true, 16 | complete: true, 17 | }), 18 | }; 19 | -------------------------------------------------------------------------------- /src/providers/lemonfox-ai/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const LemonfoxAIAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.lemonfox.ai/v1', 5 | headers: ({ providerOptions, fn }) => { 6 | const headersObj: Record = { 7 | Authorization: `Bearer ${providerOptions.apiKey}`, 8 | }; 9 | if (fn === 'createTranscription') 10 | headersObj['content-type'] = 'multipart/form-data'; 11 | return headersObj; 12 | }, 13 | getEndpoint: ({ fn }) => { 14 | switch (fn) { 15 | case 'chatComplete': 16 | return '/chat/completions'; 17 | case 'imageGenerate': 18 | return '/images/generations'; 19 | case 'createTranscription': 20 | return '/audio/transcriptions'; 21 | default: 22 | return ''; 23 | } 24 | }, 25 | }; 26 | 27 | export default LemonfoxAIAPIConfig; 28 | -------------------------------------------------------------------------------- /src/providers/lemonfox-ai/chatComplete.ts: -------------------------------------------------------------------------------- 1 | import { LEMONFOX_AI } from '../../globals'; 2 | 3 | interface LemonfoxAIStreamChunk { 4 | id: string; 5 | object: string; 6 | created: number; 7 | model: string; 8 | choices: { 9 | delta: { 10 | role?: string | null; 11 | content?: string; 12 | }; 13 | index: number; 14 | finish_reason: string | null; 15 | }[]; 16 | } 17 | 18 | export const LemonfoxAIChatCompleteStreamChunkTransform: ( 19 | response: string 20 | ) => string = (responseChunk) => { 21 | let chunk = responseChunk.trim(); 22 | chunk = chunk.replace(/^data: /, ''); 23 | chunk = chunk.trim(); 24 | if (chunk === '[DONE]') { 25 | return `data: ${chunk}\n\n`; 26 | } 27 | const parsedChunk: LemonfoxAIStreamChunk = JSON.parse(chunk); 28 | return ( 29 | `data: ${JSON.stringify({ 30 | id: parsedChunk.id, 31 | object: parsedChunk.object, 32 | created: parsedChunk.created, 33 | model: parsedChunk.model, 34 | provider: LEMONFOX_AI, 35 | choices: [ 36 | { 37 | index: parsedChunk.choices[0].index, 38 | delta: parsedChunk.choices[0].delta, 39 | finish_reason: parsedChunk.choices[0].finish_reason, 40 | }, 41 | ], 42 | })}` + '\n\n' 43 | ); 44 | }; 45 | -------------------------------------------------------------------------------- /src/providers/lemonfox-ai/createTranscription.ts: -------------------------------------------------------------------------------- 1 | import { LEMONFOX_AI } from '../../globals'; 2 | 3 | import { generateInvalidProviderResponseError } from '../utils'; 4 | 5 | import { ErrorResponse, ProviderConfig } from '../types'; 6 | 7 | export const LemonfoxAIcreateTranscriptionConfig: ProviderConfig = { 8 | file: { 9 | param: 'file', 10 | required: true, 11 | }, 12 | response_format: { 13 | param: 'response_format', 14 | }, 15 | prompt: { 16 | param: 'prompt', 17 | }, 18 | language: { 19 | param: 'language', 20 | }, 21 | translate: { 22 | param: 'translate', 23 | }, 24 | }; 25 | 26 | export const LemonfoxAICreateTranscriptionResponseTransform: ( 27 | response: Response | ErrorResponse, 28 | responseStatus: number 29 | ) => Response | ErrorResponse = (response, responseStatus) => { 30 | if (responseStatus !== 200 && 'error' in response) { 31 | return generateInvalidProviderResponseError(response, LEMONFOX_AI); 32 | } 33 | return response; 34 | }; 35 | -------------------------------------------------------------------------------- /src/providers/lepton/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const LEPTON_API_URL = 'https://api.lepton.ai'; 4 | 5 | const LeptonAPIConfig: ProviderAPIConfig = { 6 | getBaseURL: () => LEPTON_API_URL, 7 | headers: ({ providerOptions, fn }) => { 8 | const headersObj: Record = { 9 | Authorization: `Bearer ${providerOptions.apiKey}`, 10 | }; 11 | 12 | if (fn === 'createTranscription') { 13 | headersObj['Content-Type'] = 'multipart/form-data'; 14 | } 15 | 16 | return headersObj; 17 | }, 18 | getEndpoint: ({ fn }) => { 19 | switch (fn) { 20 | case 'chatComplete': 21 | return '/api/v1/chat/completions'; 22 | case 'complete': 23 | return '/api/v1/completions'; 24 | case 'createTranscription': 25 | return '/api/v1/audio/transcriptions'; 26 | default: 27 | return ''; 28 | } 29 | }, 30 | }; 31 | 32 | export default LeptonAPIConfig; 33 | -------------------------------------------------------------------------------- /src/providers/lepton/createTranscription.ts: -------------------------------------------------------------------------------- 1 | import { LEPTON } from '../../globals'; 2 | import { ErrorResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from '../openai/utils'; 4 | 5 | export const LeptonCreateTranscriptionResponseTransform: ( 6 | response: Response | ErrorResponse, 7 | responseStatus: number 8 | ) => Response | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, LEPTON); 11 | } 12 | 13 | Object.defineProperty(response, 'provider', { 14 | value: LEPTON, 15 | enumerable: true, 16 | }); 17 | 18 | return response; 19 | }; 20 | -------------------------------------------------------------------------------- /src/providers/lepton/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import LeptonAPIConfig from './api'; 3 | import { 4 | LeptonChatCompleteConfig, 5 | LeptonChatCompleteResponseTransform, 6 | LeptonChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | import { 9 | LeptonCompleteConfig, 10 | LeptonCompleteResponseTransform, 11 | LeptonCompleteStreamChunkTransform, 12 | } from './complete'; 13 | import { LeptonCreateTranscriptionResponseTransform } from './createTranscription'; 14 | 15 | const LeptonConfig: ProviderConfigs = { 16 | chatComplete: LeptonChatCompleteConfig, 17 | complete: LeptonCompleteConfig, 18 | createTranscription: {}, 19 | api: LeptonAPIConfig, 20 | responseTransforms: { 21 | chatComplete: LeptonChatCompleteResponseTransform, 22 | 'stream-chatComplete': LeptonChatCompleteStreamChunkTransform, 23 | complete: LeptonCompleteResponseTransform, 24 | 'stream-complete': LeptonCompleteStreamChunkTransform, 25 | createTranscription: LeptonCreateTranscriptionResponseTransform, 26 | }, 27 | }; 28 | 29 | export default LeptonConfig; 30 | -------------------------------------------------------------------------------- /src/providers/lingyi/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const LingYiAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.lingyiwanwu.com', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; // https://platform.lingyiwanwu.com/apikeys 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/v1/chat/completions'; 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default LingYiAPIConfig; 19 | -------------------------------------------------------------------------------- /src/providers/lingyi/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import LingyiAPIConfig from './api'; 3 | import { 4 | LingyiChatCompleteConfig, 5 | LingyiChatCompleteResponseTransform, 6 | LingyiChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | 9 | const LingyiConfig: ProviderConfigs = { 10 | chatComplete: LingyiChatCompleteConfig, 11 | api: LingyiAPIConfig, 12 | responseTransforms: { 13 | chatComplete: LingyiChatCompleteResponseTransform, 14 | 'stream-chatComplete': LingyiChatCompleteStreamChunkTransform, 15 | }, 16 | }; 17 | 18 | export default LingyiConfig; 19 | -------------------------------------------------------------------------------- /src/providers/milvus/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const MilvusAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: ({ providerOptions }) => { 5 | return providerOptions.customHost || ''; 6 | }, 7 | headers: ({ providerOptions }) => { 8 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 9 | }, 10 | getEndpoint: ({ fn }) => { 11 | switch (fn) { 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default MilvusAPIConfig; 19 | -------------------------------------------------------------------------------- /src/providers/milvus/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import MilvusAPIConfig from './api'; 3 | 4 | const MilvusConfig: ProviderConfigs = { 5 | api: MilvusAPIConfig, 6 | responseTransforms: {}, 7 | }; 8 | 9 | export default MilvusConfig; 10 | -------------------------------------------------------------------------------- /src/providers/mistral-ai/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const MistralAIAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.mistral.ai/v1', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn, providerOptions }) => { 9 | let mappedfn = fn; 10 | if (providerOptions.mistralFimCompletion === 'true') { 11 | return '/fim/completions'; 12 | } 13 | switch (mappedfn) { 14 | case 'chatComplete': 15 | return '/chat/completions'; 16 | case 'embed': 17 | return '/embeddings'; 18 | default: 19 | return ''; 20 | } 21 | }, 22 | }; 23 | 24 | export default MistralAIAPIConfig; 25 | -------------------------------------------------------------------------------- /src/providers/mistral-ai/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import MistralAIAPIConfig from './api'; 3 | import { 4 | MistralAIChatCompleteConfig, 5 | MistralAIChatCompleteResponseTransform, 6 | MistralAIChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | import { MistralAIEmbedConfig, MistralAIEmbedResponseTransform } from './embed'; 9 | 10 | const MistralAIConfig: ProviderConfigs = { 11 | chatComplete: MistralAIChatCompleteConfig, 12 | embed: MistralAIEmbedConfig, 13 | api: MistralAIAPIConfig, 14 | responseTransforms: { 15 | chatComplete: MistralAIChatCompleteResponseTransform, 16 | 'stream-chatComplete': MistralAIChatCompleteStreamChunkTransform, 17 | embed: MistralAIEmbedResponseTransform, 18 | }, 19 | }; 20 | 21 | export default MistralAIConfig; 22 | -------------------------------------------------------------------------------- /src/providers/monsterapi/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const MonsterAPIApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://llm.monsterapi.ai/v1', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/chat/completions'; 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default MonsterAPIApiConfig; 19 | -------------------------------------------------------------------------------- /src/providers/monsterapi/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import MonsterAPIApiConfig from './api'; 3 | import { 4 | MonsterAPIChatCompleteConfig, 5 | MonsterAPIChatCompleteResponseTransform, 6 | } from './chatComplete'; 7 | 8 | const MonsterAPIConfig: ProviderConfigs = { 9 | api: MonsterAPIApiConfig, 10 | chatComplete: MonsterAPIChatCompleteConfig, 11 | responseTransforms: { 12 | chatComplete: MonsterAPIChatCompleteResponseTransform, 13 | }, 14 | }; 15 | 16 | export default MonsterAPIConfig; 17 | -------------------------------------------------------------------------------- /src/providers/moonshot/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const MoonshotAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.moonshot.cn', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; // https://platform.moonshot.cn/console/api-keys 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/v1/chat/completions'; 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default MoonshotAPIConfig; 19 | -------------------------------------------------------------------------------- /src/providers/moonshot/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import MoonshotAPIConfig from './api'; 3 | import { 4 | MoonshotChatCompleteConfig, 5 | MoonshotChatCompleteResponseTransform, 6 | MoonshotChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | 9 | const MoonshotConfig: ProviderConfigs = { 10 | chatComplete: MoonshotChatCompleteConfig, 11 | api: MoonshotAPIConfig, 12 | responseTransforms: { 13 | chatComplete: MoonshotChatCompleteResponseTransform, 14 | 'stream-chatComplete': MoonshotChatCompleteStreamChunkTransform, 15 | }, 16 | }; 17 | 18 | export default MoonshotConfig; 19 | -------------------------------------------------------------------------------- /src/providers/ncompass/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const NCompassApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.ncompass.tech/v1', 5 | headers: ({ providerOptions }) => { 6 | return { 7 | Authorization: `Bearer ${providerOptions.apiKey}`, 8 | }; 9 | }, 10 | getEndpoint: ({ fn }) => { 11 | switch (fn) { 12 | case 'chatComplete': 13 | return '/chat/completions'; 14 | default: 15 | return ''; 16 | } 17 | }, 18 | }; 19 | 20 | export default NCompassApiConfig; 21 | -------------------------------------------------------------------------------- /src/providers/ncompass/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import NCompassApiConfig from './api'; 3 | import { 4 | NCompassChatCompleteConfig, 5 | NCompassChatCompleteResponseTransform, 6 | NCompassChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | 9 | const NCompassConfig: ProviderConfigs = { 10 | chatComplete: NCompassChatCompleteConfig, 11 | api: NCompassApiConfig, 12 | responseTransforms: { 13 | chatComplete: NCompassChatCompleteResponseTransform, 14 | 'stream-chatComplete': NCompassChatCompleteStreamChunkTransform, 15 | }, 16 | }; 17 | 18 | export default NCompassConfig; 19 | -------------------------------------------------------------------------------- /src/providers/nebius/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | export const nebiusAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.studio.nebius.ai/v1', 5 | headers({ providerOptions }) { 6 | const { apiKey } = providerOptions; 7 | return { Authorization: `Bearer ${apiKey}` }; 8 | }, 9 | getEndpoint({ fn }) { 10 | switch (fn) { 11 | case 'chatComplete': 12 | return `/chat/completions`; 13 | case 'embed': 14 | return `/embeddings`; 15 | case 'complete': 16 | return '/completions'; 17 | default: 18 | return ''; 19 | } 20 | }, 21 | }; 22 | -------------------------------------------------------------------------------- /src/providers/nebius/index.ts: -------------------------------------------------------------------------------- 1 | import { NEBIUS } from '../../globals'; 2 | import { 3 | chatCompleteParams, 4 | embedParams, 5 | completeParams, 6 | responseTransformers, 7 | } from '../open-ai-base'; 8 | import { ProviderConfigs } from '../types'; 9 | import { nebiusAPIConfig } from './api'; 10 | 11 | const NebiusConfig: ProviderConfigs = { 12 | chatComplete: chatCompleteParams([], { 13 | model: 'Qwen/Qwen2.5-72B-Instruct-fast', 14 | }), 15 | embed: embedParams([], { model: 'BAAI/bge-en-icl' }), 16 | complete: completeParams([], { model: 'Qwen/Qwen2.5-72B-Instruct-fast' }), 17 | api: nebiusAPIConfig, 18 | responseTransforms: responseTransformers(NEBIUS, { 19 | chatComplete: true, 20 | embed: true, 21 | complete: true, 22 | }), 23 | }; 24 | 25 | export default NebiusConfig; 26 | -------------------------------------------------------------------------------- /src/providers/nomic/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const NomicAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api-atlas.nomic.ai/v1', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'embed': 11 | return '/embedding/text'; 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default NomicAPIConfig; 19 | -------------------------------------------------------------------------------- /src/providers/nomic/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import NomicAPIConfig from './api'; 3 | import { NomicEmbedConfig, NomicEmbedResponseTransform } from './embed'; 4 | 5 | const NomicConfig: ProviderConfigs = { 6 | embed: NomicEmbedConfig, 7 | api: NomicAPIConfig, 8 | responseTransforms: { 9 | embed: NomicEmbedResponseTransform, 10 | }, 11 | }; 12 | 13 | export default NomicConfig; 14 | -------------------------------------------------------------------------------- /src/providers/novita-ai/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const NovitaAIApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.novita.ai/v3/openai', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'complete': 11 | return '/v1/completions'; 12 | case 'chatComplete': 13 | return '/v1/chat/completions'; 14 | default: 15 | return ''; 16 | } 17 | }, 18 | }; 19 | 20 | export default NovitaAIApiConfig; 21 | -------------------------------------------------------------------------------- /src/providers/novita-ai/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import NovitaAIApiConfig from './api'; 3 | import { 4 | NovitaAIChatCompleteConfig, 5 | NovitaAIChatCompleteResponseTransform, 6 | NovitaAIChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | import { 9 | NovitaAICompleteConfig, 10 | NovitaAICompleteResponseTransform, 11 | NovitaAICompleteStreamChunkTransform, 12 | } from './complete'; 13 | 14 | const NovitaAIConfig: ProviderConfigs = { 15 | complete: NovitaAICompleteConfig, 16 | chatComplete: NovitaAIChatCompleteConfig, 17 | api: NovitaAIApiConfig, 18 | responseTransforms: { 19 | 'stream-complete': NovitaAICompleteStreamChunkTransform, 20 | complete: NovitaAICompleteResponseTransform, 21 | chatComplete: NovitaAIChatCompleteResponseTransform, 22 | 'stream-chatComplete': NovitaAIChatCompleteStreamChunkTransform, 23 | }, 24 | }; 25 | 26 | export default NovitaAIConfig; 27 | -------------------------------------------------------------------------------- /src/providers/nscale/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const NscaleAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://inference.api.nscale.com/v1', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/chat/completions'; 12 | case 'imageGenerate': 13 | return '/images/generations'; 14 | default: 15 | return ''; 16 | } 17 | }, 18 | }; 19 | 20 | export default NscaleAPIConfig; 21 | -------------------------------------------------------------------------------- /src/providers/nscale/imageGenerate.ts: -------------------------------------------------------------------------------- 1 | import { ParameterConfig } from '../types'; 2 | 3 | export const NscaleImageGenerateConfig: { [key: string]: ParameterConfig } = { 4 | prompt: { 5 | param: 'prompt', 6 | required: true, 7 | }, 8 | model: { 9 | param: 'model', 10 | required: true, 11 | }, 12 | n: { 13 | param: 'n', 14 | }, 15 | size: { 16 | param: 'size', 17 | }, 18 | }; 19 | 20 | export const NscaleImageGenerateResponseTransform = (response: any) => { 21 | return { 22 | created: Date.now(), 23 | data: response.data.map((item: any) => ({ 24 | url: item.url, 25 | b64_json: item.b64_json, 26 | })), 27 | }; 28 | }; 29 | -------------------------------------------------------------------------------- /src/providers/nscale/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import NscaleAPIConfig from './api'; 3 | import { 4 | NscaleImageGenerateConfig, 5 | NscaleImageGenerateResponseTransform, 6 | } from './imageGenerate'; 7 | import { responseTransformers } from '../open-ai-base'; 8 | import { NSCALE } from '../../globals'; 9 | import { chatCompleteParams } from '../open-ai-base'; 10 | 11 | const NscaleConfig: ProviderConfigs = { 12 | chatComplete: chatCompleteParams([ 13 | 'functions', 14 | 'function_call', 15 | 'user', 16 | 'seed', 17 | 'tools', 18 | 'tool_choice', 19 | 'stream_options', 20 | ]), 21 | imageGenerate: NscaleImageGenerateConfig, 22 | api: NscaleAPIConfig, 23 | responseTransforms: { 24 | ...responseTransformers(NSCALE, { chatComplete: true }), 25 | imageGenerate: NscaleImageGenerateResponseTransform, 26 | }, 27 | }; 28 | 29 | export default NscaleConfig; 30 | -------------------------------------------------------------------------------- /src/providers/ollama/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const OllamaAPIConfig: ProviderAPIConfig = { 4 | headers: () => { 5 | return {}; 6 | }, 7 | getBaseURL: ({ providerOptions }) => { 8 | return providerOptions.customHost ?? ''; 9 | }, 10 | getEndpoint: ({ fn, providerOptions }) => { 11 | let mappedFn = fn; 12 | const { urlToFetch } = providerOptions; 13 | if (fn === 'proxy' && urlToFetch && urlToFetch?.indexOf('/api/chat') > -1) { 14 | mappedFn = 'chatComplete'; 15 | } else if ( 16 | fn === 'proxy' && 17 | urlToFetch && 18 | urlToFetch?.indexOf('/embeddings') > -1 19 | ) { 20 | mappedFn = 'embed'; 21 | } 22 | 23 | switch (mappedFn) { 24 | case 'chatComplete': { 25 | return `/v1/chat/completions`; 26 | } 27 | case 'embed': { 28 | return `/api/embeddings`; 29 | } 30 | default: 31 | return ''; 32 | } 33 | }, 34 | }; 35 | 36 | export default OllamaAPIConfig; 37 | -------------------------------------------------------------------------------- /src/providers/ollama/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import { OllamaEmbedConfig, OllamaEmbedResponseTransform } from './embed'; 3 | import OllamaAPIConfig from './api'; 4 | import { 5 | OllamaChatCompleteConfig, 6 | OllamaChatCompleteResponseTransform, 7 | OllamaChatCompleteStreamChunkTransform, 8 | } from './chatComplete'; 9 | 10 | const OllamaConfig: ProviderConfigs = { 11 | embed: OllamaEmbedConfig, 12 | api: OllamaAPIConfig, 13 | chatComplete: OllamaChatCompleteConfig, 14 | responseTransforms: { 15 | chatComplete: OllamaChatCompleteResponseTransform, 16 | 'stream-chatComplete': OllamaChatCompleteStreamChunkTransform, 17 | embed: OllamaEmbedResponseTransform, 18 | }, 19 | }; 20 | 21 | export default OllamaConfig; 22 | -------------------------------------------------------------------------------- /src/providers/openai/cancelBatch.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { CancelBatchResponse, ErrorResponse, ProviderConfig } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAICancelBatchResponseTransform: ( 6 | response: CancelBatchResponse | ErrorResponse, 7 | responseStatus: number 8 | ) => CancelBatchResponse | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/openai/createBatch.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse, ProviderConfig, CreateBatchResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAICreateBatchConfig: ProviderConfig = { 6 | input_file_id: { 7 | param: 'input_file_id', 8 | required: true, 9 | }, 10 | endpoint: { 11 | param: 'endpoint', 12 | required: true, 13 | }, 14 | completion_window: { 15 | param: 'completion_window', 16 | default: '24h', 17 | required: true, 18 | }, 19 | metadata: { 20 | param: 'metadata', 21 | required: false, 22 | }, 23 | }; 24 | 25 | export const OpenAICreateBatchResponseTransform: ( 26 | response: CreateBatchResponse | ErrorResponse, 27 | responseStatus: number 28 | ) => CreateBatchResponse | ErrorResponse = (response, responseStatus) => { 29 | if (responseStatus !== 200 && 'error' in response) { 30 | return OpenAIErrorResponseTransform(response, OPEN_AI); 31 | } 32 | 33 | return response; 34 | }; 35 | -------------------------------------------------------------------------------- /src/providers/openai/createFinetune.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse, ProviderConfig } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAICreateFinetuneConfig: ProviderConfig = { 6 | model: { 7 | param: 'model', 8 | required: true, 9 | }, 10 | suffix: { 11 | param: 'suffix', 12 | required: true, 13 | }, 14 | hyperparameters: { 15 | param: 'hyperparameters', 16 | required: false, 17 | }, 18 | training_file: { 19 | param: 'training_file', 20 | required: true, 21 | }, 22 | validation_file: { 23 | param: 'validation_file', 24 | required: false, 25 | }, 26 | integrations: { 27 | param: 'integrations', 28 | required: false, 29 | }, 30 | seed: { 31 | param: 'seed', 32 | required: false, 33 | }, 34 | method: { 35 | param: 'method', 36 | required: false, 37 | }, 38 | }; 39 | 40 | export const OpenAIFinetuneResponseTransform: ( 41 | response: Response | ErrorResponse, 42 | responseStatus: number 43 | ) => Response | ErrorResponse = (response, responseStatus) => { 44 | if (responseStatus !== 200 && 'error' in response) { 45 | return OpenAIErrorResponseTransform(response, OPEN_AI); 46 | } 47 | 48 | return response; 49 | }; 50 | -------------------------------------------------------------------------------- /src/providers/openai/createSpeech.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse, ProviderConfig } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAICreateSpeechConfig: ProviderConfig = { 6 | model: { 7 | param: 'model', 8 | required: true, 9 | default: 'tts-1', 10 | }, 11 | input: { 12 | param: 'input', 13 | required: true, 14 | }, 15 | voice: { 16 | param: 'voice', 17 | required: true, 18 | default: 'alloy', 19 | }, 20 | response_format: { 21 | param: 'response_format', 22 | required: false, 23 | default: 'mp3', 24 | }, 25 | speed: { 26 | param: 'speed', 27 | required: false, 28 | default: 1, 29 | }, 30 | }; 31 | 32 | export const OpenAICreateSpeechResponseTransform: ( 33 | response: Response | ErrorResponse, 34 | responseStatus: number 35 | ) => Response | ErrorResponse = (response, responseStatus) => { 36 | if (responseStatus !== 200 && 'error' in response) { 37 | return OpenAIErrorResponseTransform(response, OPEN_AI); 38 | } 39 | 40 | return response; 41 | }; 42 | -------------------------------------------------------------------------------- /src/providers/openai/createTranscription.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAICreateTranscriptionResponseTransform: ( 6 | response: Response | ErrorResponse, 7 | responseStatus: number 8 | ) => Response | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/openai/createTranslation.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAICreateTranslationResponseTransform: ( 6 | response: Response | ErrorResponse, 7 | responseStatus: number 8 | ) => Response | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/openai/deleteFile.ts: -------------------------------------------------------------------------------- 1 | import { ErrorResponse } from '../types'; 2 | import { OPEN_AI } from '../../globals'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAIDeleteFileResponseTransform: ( 6 | response: Response | ErrorResponse, 7 | responseStatus: number 8 | ) => Response | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/openai/listBatches.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse, ListBatchesResponse, ProviderConfig } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAIListBatchesResponseTransform: ( 6 | response: ListBatchesResponse | ErrorResponse, 7 | responseStatus: number 8 | ) => ListBatchesResponse | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/openai/listFiles.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAIGetFilesResponseTransform = ( 6 | response: Response | ErrorResponse, 7 | responseStatus: number 8 | ): Response | ErrorResponse => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/openai/retrieveBatch.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse, ProviderConfig, RetrieveBatchResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAIRetrieveBatchResponseTransform: ( 6 | response: RetrieveBatchResponse | ErrorResponse, 7 | responseStatus: number 8 | ) => RetrieveBatchResponse | ErrorResponse = (response, responseStatus) => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/openai/retrieveFileContent.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAIGetFileContentResponseTransform = ( 6 | response: Response | ErrorResponse, 7 | responseStatus: number 8 | ): Response | ErrorResponse => { 9 | if (responseStatus !== 200 && 'error' in response) { 10 | return OpenAIErrorResponseTransform(response, OPEN_AI); 11 | } 12 | 13 | return response; 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/openai/uploadFile.ts: -------------------------------------------------------------------------------- 1 | import { OPEN_AI } from '../../globals'; 2 | import { ErrorResponse } from '../types'; 3 | import { OpenAIErrorResponseTransform } from './utils'; 4 | 5 | export const OpenAIFileUploadRequestTransform = (requestBody: FormData) => { 6 | return requestBody; 7 | }; 8 | 9 | export const OpenAIUploadFileResponseTransform: ( 10 | response: Response | ErrorResponse, 11 | responseStatus: number 12 | ) => Response | ErrorResponse = (response, responseStatus) => { 13 | if (responseStatus !== 200 && 'error' in response) { 14 | return OpenAIErrorResponseTransform(response, OPEN_AI); 15 | } 16 | 17 | return response; 18 | }; 19 | -------------------------------------------------------------------------------- /src/providers/openai/utils.ts: -------------------------------------------------------------------------------- 1 | import { ErrorResponse } from '../types'; 2 | import { generateErrorResponse } from '../utils'; 3 | 4 | export const OpenAIErrorResponseTransform: ( 5 | response: ErrorResponse, 6 | provider: string 7 | ) => ErrorResponse = (response, provider) => { 8 | return generateErrorResponse( 9 | { 10 | ...response.error, 11 | }, 12 | provider 13 | ); 14 | }; 15 | -------------------------------------------------------------------------------- /src/providers/openrouter/api.ts: -------------------------------------------------------------------------------- 1 | import { POWERED_BY } from '../../globals'; 2 | import { ProviderAPIConfig } from '../types'; 3 | 4 | const OpenrouterAPIConfig: ProviderAPIConfig = { 5 | getBaseURL: () => 'https://openrouter.ai/api', 6 | headers: ({ providerOptions }) => { 7 | return { 8 | Authorization: `Bearer ${providerOptions.apiKey}`, // https://openrouter.ai/keys 9 | 'HTTP-Referer': 'https://portkey.ai/', 10 | 'X-Title': POWERED_BY, 11 | }; 12 | }, 13 | getEndpoint: ({ fn }) => { 14 | switch (fn) { 15 | case 'chatComplete': 16 | return '/v1/chat/completions'; 17 | default: 18 | return ''; 19 | } 20 | }, 21 | }; 22 | 23 | export default OpenrouterAPIConfig; 24 | -------------------------------------------------------------------------------- /src/providers/openrouter/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import OpenrouterAPIConfig from './api'; 3 | import { 4 | OpenrouterChatCompleteConfig, 5 | OpenrouterChatCompleteResponseTransform, 6 | OpenrouterChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | 9 | const OpenrouterConfig: ProviderConfigs = { 10 | chatComplete: OpenrouterChatCompleteConfig, 11 | api: OpenrouterAPIConfig, 12 | responseTransforms: { 13 | chatComplete: OpenrouterChatCompleteResponseTransform, 14 | 'stream-chatComplete': OpenrouterChatCompleteStreamChunkTransform, 15 | }, 16 | }; 17 | 18 | export default OpenrouterConfig; 19 | -------------------------------------------------------------------------------- /src/providers/palm/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | export const PalmApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://generativelanguage.googleapis.com/v1beta3', 5 | headers: () => { 6 | return { 'Content-Type': 'application/json' }; 7 | }, 8 | getEndpoint: ({ providerOptions, fn, gatewayRequestBodyJSON }) => { 9 | const { apiKey } = providerOptions; 10 | const { model } = gatewayRequestBodyJSON; 11 | switch (fn) { 12 | case 'complete': { 13 | return `/models/${model}:generateText?key=${apiKey}`; 14 | } 15 | case 'chatComplete': { 16 | return `/models/${model}:generateMessage?key=${apiKey}`; 17 | } 18 | case 'embed': { 19 | return `/models/${model}:embedText?key=${apiKey}`; 20 | } 21 | default: 22 | return ''; 23 | } 24 | }, 25 | }; 26 | 27 | export default PalmApiConfig; 28 | -------------------------------------------------------------------------------- /src/providers/palm/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import PalmApiConfig from './api'; 3 | import { 4 | PalmChatCompleteConfig, 5 | PalmChatCompleteResponseTransform, 6 | } from './chatComplete'; 7 | import { PalmCompleteConfig, PalmCompleteResponseTransform } from './complete'; 8 | import { PalmEmbedConfig, PalmEmbedResponseTransform } from './embed'; 9 | 10 | const PalmAIConfig: ProviderConfigs = { 11 | complete: PalmCompleteConfig, 12 | embed: PalmEmbedConfig, 13 | api: PalmApiConfig, 14 | chatComplete: PalmChatCompleteConfig, 15 | responseTransforms: { 16 | complete: PalmCompleteResponseTransform, 17 | chatComplete: PalmChatCompleteResponseTransform, 18 | embed: PalmEmbedResponseTransform, 19 | }, 20 | }; 21 | 22 | export default PalmAIConfig; 23 | -------------------------------------------------------------------------------- /src/providers/perplexity-ai/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const PerplexityAIApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.perplexity.ai', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/chat/completions'; 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default PerplexityAIApiConfig; 19 | -------------------------------------------------------------------------------- /src/providers/perplexity-ai/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import PerplexityAIApiConfig from './api'; 3 | import { 4 | PerplexityAIChatCompleteConfig, 5 | PerplexityAIChatCompleteResponseTransform, 6 | PerplexityAIChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | 9 | const PerplexityAIConfig: ProviderConfigs = { 10 | chatComplete: PerplexityAIChatCompleteConfig, 11 | api: PerplexityAIApiConfig, 12 | responseTransforms: { 13 | chatComplete: PerplexityAIChatCompleteResponseTransform, 14 | 'stream-chatComplete': PerplexityAIChatCompleteStreamChunkTransform, 15 | }, 16 | }; 17 | 18 | export default PerplexityAIConfig; 19 | -------------------------------------------------------------------------------- /src/providers/predibase/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | import { splitString } from '../utils'; 3 | 4 | const PredibaseAPIConfig: ProviderAPIConfig = { 5 | getBaseURL: () => 'https://serving.app.predibase.com', 6 | headers: ({ providerOptions }) => { 7 | return { 8 | Authorization: `Bearer ${providerOptions.apiKey}`, 9 | Accept: 'application/json', 10 | }; 11 | }, 12 | getEndpoint: ({ fn, gatewayRequestBodyJSON }) => { 13 | const user = gatewayRequestBodyJSON?.user; 14 | const model = gatewayRequestBodyJSON?.model; 15 | const base_model = splitString(`${model}`, ':').before; 16 | switch (fn) { 17 | case 'chatComplete': 18 | /* 19 | The Predibase model format is "[:adapter_id]", 20 | where adapter_id format is " { 5 | return providerOptions.customHost || ''; 6 | }, 7 | headers: ({ providerOptions }) => { 8 | return { 'api-key': `Bearer ${providerOptions.apiKey}` }; 9 | }, 10 | getEndpoint: ({ fn }) => { 11 | switch (fn) { 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default QdrantAPIConfig; 19 | -------------------------------------------------------------------------------- /src/providers/qdrant/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import QdrantAPIConfig from './api'; 3 | 4 | const QdrantConfig: ProviderConfigs = { 5 | api: QdrantAPIConfig, 6 | responseTransforms: {}, 7 | }; 8 | 9 | export default QdrantConfig; 10 | -------------------------------------------------------------------------------- /src/providers/recraft-ai/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const RecraftAIAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://external.api.recraft.ai/v1', 5 | headers: ({ providerOptions }) => ({ 6 | Authorization: `Bearer ${providerOptions.apiKey}`, 7 | 'Content-Type': 'application/json', 8 | }), 9 | getEndpoint: ({ fn }) => { 10 | switch (fn) { 11 | case 'imageGenerate': 12 | return '/images/generations'; 13 | default: 14 | return ''; 15 | } 16 | }, 17 | }; 18 | 19 | export default RecraftAIAPIConfig; 20 | -------------------------------------------------------------------------------- /src/providers/recraft-ai/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import RecraftAIAPIConfig from './api'; 3 | import { 4 | RecraftAIImageGenerateConfig, 5 | RecraftAIImageGenerateResponseTransform, 6 | } from './imageGenerate'; 7 | 8 | const RecraftAIConfig: ProviderConfigs = { 9 | imageGenerate: RecraftAIImageGenerateConfig, 10 | api: RecraftAIAPIConfig, 11 | responseTransforms: { 12 | imageGenerate: RecraftAIImageGenerateResponseTransform, 13 | }, 14 | }; 15 | 16 | export default RecraftAIConfig; 17 | -------------------------------------------------------------------------------- /src/providers/recraft-ai/utils.ts: -------------------------------------------------------------------------------- 1 | import { ErrorResponse } from '../types'; 2 | import { generateErrorResponse } from '../utils'; 3 | 4 | export const RecraftAIErrorResponseTransform: ( 5 | response: ErrorResponse, 6 | provider: string 7 | ) => ErrorResponse = (response, provider) => { 8 | return generateErrorResponse( 9 | { 10 | message: response.error?.message || 'Unknown error occurred', 11 | type: response.error?.type || null, 12 | param: response.error?.param || null, 13 | code: response.error?.code || null, 14 | }, 15 | provider 16 | ); 17 | }; 18 | -------------------------------------------------------------------------------- /src/providers/reka-ai/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const RekaAIApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.reka.ai', 5 | headers: ({ providerOptions }) => { 6 | return { 'x-api-key': `${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/chat'; 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default RekaAIApiConfig; 19 | -------------------------------------------------------------------------------- /src/providers/reka-ai/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import RekaAIApiConfig from './api'; 3 | import { 4 | RekaAIChatCompleteConfig, 5 | RekaAIChatCompleteResponseTransform, 6 | } from './chatComplete'; 7 | 8 | const RekaAIConfig: ProviderConfigs = { 9 | chatComplete: RekaAIChatCompleteConfig, 10 | api: RekaAIApiConfig, 11 | responseTransforms: { 12 | chatComplete: RekaAIChatCompleteResponseTransform, 13 | }, 14 | }; 15 | 16 | export default RekaAIConfig; 17 | -------------------------------------------------------------------------------- /src/providers/replicate/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const ReplicateApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.replicate.com/v1', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => '', 9 | }; 10 | 11 | export default ReplicateApiConfig; 12 | -------------------------------------------------------------------------------- /src/providers/replicate/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import ReplicateAPIConfig from './api'; 3 | 4 | const ReplicateConfig: ProviderConfigs = { 5 | api: ReplicateAPIConfig, 6 | responseTransforms: {}, 7 | }; 8 | 9 | export default ReplicateConfig; 10 | -------------------------------------------------------------------------------- /src/providers/sagemaker/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import SagemakerAPIConfig from './api'; 3 | 4 | const SagemakerConfig: ProviderConfigs = { 5 | api: SagemakerAPIConfig, 6 | }; 7 | 8 | export default SagemakerConfig; 9 | -------------------------------------------------------------------------------- /src/providers/sambanova/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const SambaNovaAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: ({ providerOptions }) => 5 | providerOptions.customHost || 'https://api.sambanova.ai', 6 | headers: ({ providerOptions }) => { 7 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 8 | }, 9 | getEndpoint: ({ fn }) => { 10 | switch (fn) { 11 | case 'chatComplete': 12 | return '/v1/chat/completions'; 13 | default: 14 | return ''; 15 | } 16 | }, 17 | }; 18 | 19 | export default SambaNovaAPIConfig; 20 | -------------------------------------------------------------------------------- /src/providers/segmind/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const SegmindAIAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.segmind.com/v1', 5 | headers: ({ providerOptions }) => { 6 | return { 'x-api-key': `${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ gatewayRequestBodyJSON }) => { 9 | return `/${gatewayRequestBodyJSON.model}`; 10 | }, 11 | }; 12 | 13 | export default SegmindAIAPIConfig; 14 | -------------------------------------------------------------------------------- /src/providers/segmind/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import SegmindAIAPIConfig from './api'; 3 | import { 4 | SegmindImageGenerateConfig, 5 | SegmindImageGenerateResponseTransform, 6 | } from './imageGenerate'; 7 | 8 | const SegmindConfig: ProviderConfigs = { 9 | api: SegmindAIAPIConfig, 10 | imageGenerate: SegmindImageGenerateConfig, 11 | responseTransforms: { 12 | imageGenerate: SegmindImageGenerateResponseTransform, 13 | }, 14 | }; 15 | 16 | export default SegmindConfig; 17 | -------------------------------------------------------------------------------- /src/providers/siliconflow/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const SiliconFlowAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.siliconflow.cn/v1', 5 | headers: ({ providerOptions }) => { 6 | return { 7 | Authorization: `Bearer ${providerOptions.apiKey}`, 8 | }; 9 | }, 10 | getEndpoint: ({ fn, gatewayRequestBodyJSON }) => { 11 | const { model = 'ByteDance/SDXL-Lightning' } = gatewayRequestBodyJSON; 12 | switch (fn) { 13 | case 'chatComplete': 14 | return '/chat/completions'; 15 | case 'embed': 16 | return '/embeddings'; 17 | case 'imageGenerate': 18 | return `/${model}/text-to-image`; 19 | default: 20 | return ''; 21 | } 22 | }, 23 | }; 24 | 25 | export default SiliconFlowAPIConfig; 26 | -------------------------------------------------------------------------------- /src/providers/siliconflow/embed.ts: -------------------------------------------------------------------------------- 1 | import { SILICONFLOW } from '../../globals'; 2 | import { EmbedResponse } from '../../types/embedRequestBody'; 3 | import { ErrorResponse, ProviderConfig } from '../types'; 4 | import { SiliconFlowErrorResponseTransform } from './chatComplete'; 5 | 6 | // TODOS: this configuration does not enforce the maximum token limit for the input parameter. If you want to enforce this, you might need to add a custom validation function or a max property to the ParameterConfig interface, and then use it in the input configuration. However, this might be complex because the token count is not a simple length check, but depends on the specific tokenization method used by the model. 7 | 8 | export const SiliconFlowEmbedConfig: ProviderConfig = { 9 | model: { 10 | param: 'model', 11 | required: true, 12 | default: 'BAAI/bge-large-zh-v1.5', 13 | }, 14 | input: { 15 | param: 'input', 16 | required: true, 17 | }, 18 | encoding_format: { 19 | param: 'encoding_format', 20 | }, 21 | }; 22 | 23 | interface SiliconFlowEmbedResponse extends EmbedResponse {} 24 | 25 | export const SiliconFlowEmbedResponseTransform: ( 26 | response: SiliconFlowEmbedResponse | ErrorResponse, 27 | responseStatus: number 28 | ) => EmbedResponse | ErrorResponse = (response, responseStatus) => { 29 | if (responseStatus !== 200 && 'error' in response) { 30 | return SiliconFlowErrorResponseTransform(response, SILICONFLOW); 31 | } 32 | 33 | return response; 34 | }; 35 | -------------------------------------------------------------------------------- /src/providers/siliconflow/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import { 3 | SiliconFlowEmbedConfig, 4 | SiliconFlowEmbedResponseTransform, 5 | } from './embed'; 6 | import SiliconFlowAPIConfig from './api'; 7 | import { 8 | SiliconFlowChatCompleteConfig, 9 | SiliconFlowChatCompleteResponseTransform, 10 | } from './chatComplete'; 11 | import { 12 | SiliconFlowImageGenerateConfig, 13 | SiliconFlowImageGenerateResponseTransform, 14 | } from './imageGenerate'; 15 | 16 | const SiliconFlowConfig: ProviderConfigs = { 17 | embed: SiliconFlowEmbedConfig, 18 | api: SiliconFlowAPIConfig, 19 | chatComplete: SiliconFlowChatCompleteConfig, 20 | imageGenerate: SiliconFlowImageGenerateConfig, 21 | responseTransforms: { 22 | chatComplete: SiliconFlowChatCompleteResponseTransform, 23 | embed: SiliconFlowEmbedResponseTransform, 24 | imageGenerate: SiliconFlowImageGenerateResponseTransform, 25 | }, 26 | }; 27 | 28 | export default SiliconFlowConfig; 29 | -------------------------------------------------------------------------------- /src/providers/stability-ai/constants.ts: -------------------------------------------------------------------------------- 1 | export const STABILITY_V1_MODELS = [ 2 | 'stable-diffusion-xl-1024-v1-0', 3 | 'stable-diffusion-v1-6', 4 | ]; 5 | -------------------------------------------------------------------------------- /src/providers/stability-ai/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import StabilityAIAPIConfig from './api'; 3 | import { STABILITY_V1_MODELS } from './constants'; 4 | import { 5 | StabilityAIImageGenerateV1Config, 6 | StabilityAIImageGenerateV1ResponseTransform, 7 | } from './imageGenerate'; 8 | import { 9 | StabilityAIImageGenerateV2Config, 10 | StabilityAIImageGenerateV2ResponseTransform, 11 | } from './imageGenerateV2'; 12 | import { isStabilityV1Model } from './utils'; 13 | 14 | const StabilityAIConfig: ProviderConfigs = { 15 | api: StabilityAIAPIConfig, 16 | getConfig: (params: Params) => { 17 | const model = params.model; 18 | if (typeof model === 'string' && isStabilityV1Model(model)) { 19 | return { 20 | imageGenerate: StabilityAIImageGenerateV1Config, 21 | responseTransforms: { 22 | imageGenerate: StabilityAIImageGenerateV1ResponseTransform, 23 | }, 24 | }; 25 | } 26 | return { 27 | imageGenerate: StabilityAIImageGenerateV2Config, 28 | responseTransforms: { 29 | imageGenerate: StabilityAIImageGenerateV2ResponseTransform, 30 | }, 31 | }; 32 | }, 33 | }; 34 | 35 | export default StabilityAIConfig; 36 | -------------------------------------------------------------------------------- /src/providers/stability-ai/utils.ts: -------------------------------------------------------------------------------- 1 | import { STABILITY_V1_MODELS } from './constants'; 2 | 3 | export const isStabilityV1Model = (model?: string) => { 4 | if (!model) return false; 5 | return STABILITY_V1_MODELS.includes(model); 6 | }; 7 | -------------------------------------------------------------------------------- /src/providers/together-ai/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const TogetherAIApiConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.together.xyz', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'complete': 11 | return '/v1/completions'; 12 | case 'chatComplete': 13 | return '/v1/chat/completions'; 14 | case 'embed': 15 | return '/v1/embeddings'; 16 | default: 17 | return ''; 18 | } 19 | }, 20 | }; 21 | 22 | export default TogetherAIApiConfig; 23 | -------------------------------------------------------------------------------- /src/providers/together-ai/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import TogetherAIApiConfig from './api'; 3 | import { 4 | TogetherAIChatCompleteConfig, 5 | TogetherAIChatCompleteResponseTransform, 6 | TogetherAIChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | import { 9 | TogetherAICompleteConfig, 10 | TogetherAICompleteResponseTransform, 11 | TogetherAICompleteStreamChunkTransform, 12 | } from './complete'; 13 | import { 14 | TogetherAIEmbedConfig, 15 | TogetherAIEmbedResponseTransform, 16 | } from './embed'; 17 | 18 | const TogetherAIConfig: ProviderConfigs = { 19 | complete: TogetherAICompleteConfig, 20 | chatComplete: TogetherAIChatCompleteConfig, 21 | embed: TogetherAIEmbedConfig, 22 | api: TogetherAIApiConfig, 23 | responseTransforms: { 24 | 'stream-complete': TogetherAICompleteStreamChunkTransform, 25 | complete: TogetherAICompleteResponseTransform, 26 | chatComplete: TogetherAIChatCompleteResponseTransform, 27 | 'stream-chatComplete': TogetherAIChatCompleteStreamChunkTransform, 28 | embed: TogetherAIEmbedResponseTransform, 29 | }, 30 | }; 31 | 32 | export default TogetherAIConfig; 33 | -------------------------------------------------------------------------------- /src/providers/triton/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const TritonAPIConfig: ProviderAPIConfig = { 4 | headers: () => { 5 | return {}; 6 | }, 7 | getBaseURL: ({ providerOptions }) => { 8 | return providerOptions.customHost ?? ''; 9 | }, 10 | getEndpoint: ({ fn, providerOptions }) => { 11 | let mappedFn = fn; 12 | switch (mappedFn) { 13 | case 'complete': { 14 | return `/generate`; 15 | } 16 | default: 17 | return ''; 18 | } 19 | }, 20 | }; 21 | 22 | export default TritonAPIConfig; 23 | -------------------------------------------------------------------------------- /src/providers/triton/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import TritonAPIConfig from './api'; 3 | import { 4 | TritonCompleteConfig, 5 | TritonCompleteResponseTransform, 6 | } from './complete'; 7 | 8 | const TritonConfig: ProviderConfigs = { 9 | api: TritonAPIConfig, 10 | complete: TritonCompleteConfig, 11 | responseTransforms: { 12 | complete: TritonCompleteResponseTransform, 13 | }, 14 | }; 15 | 16 | export default TritonConfig; 17 | -------------------------------------------------------------------------------- /src/providers/upstage/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | export const upstageAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.upstage.ai/v1/solar', 5 | headers({ providerOptions }) { 6 | const { apiKey } = providerOptions; 7 | return { Authorization: `Bearer ${apiKey}` }; 8 | }, 9 | getEndpoint({ fn }) { 10 | switch (fn) { 11 | case 'chatComplete': 12 | return `/chat/completions`; 13 | case 'embed': 14 | return `/embeddings`; 15 | default: 16 | return ''; 17 | } 18 | }, 19 | }; 20 | -------------------------------------------------------------------------------- /src/providers/upstage/index.ts: -------------------------------------------------------------------------------- 1 | import { UPSTAGE } from '../../globals'; 2 | import { 3 | chatCompleteParams, 4 | embedParams, 5 | responseTransformers, 6 | } from '../open-ai-base'; 7 | import { ProviderConfigs } from '../types'; 8 | import { upstageAPIConfig } from './api'; 9 | 10 | export const UpstageConfig: ProviderConfigs = { 11 | chatComplete: chatCompleteParams([], { model: 'solar-pro' }), 12 | embed: embedParams([], { model: 'solar-embedding-1-large-query' }), 13 | api: upstageAPIConfig, 14 | responseTransforms: responseTransformers(UPSTAGE, { 15 | chatComplete: true, 16 | embed: true, 17 | }), 18 | }; 19 | -------------------------------------------------------------------------------- /src/providers/voyage/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const VoyageAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.voyageai.com/v1', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'embed': 11 | return '/embeddings'; 12 | default: 13 | return ''; 14 | } 15 | }, 16 | }; 17 | 18 | export default VoyageAPIConfig; 19 | -------------------------------------------------------------------------------- /src/providers/voyage/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import VoyageAPIConfig from './api'; 3 | import { VoyageEmbedConfig, VoyageEmbedResponseTransform } from './embed'; 4 | 5 | const VoyageConfig: ProviderConfigs = { 6 | embed: VoyageEmbedConfig, 7 | api: VoyageAPIConfig, 8 | responseTransforms: { 9 | embed: VoyageEmbedResponseTransform, 10 | }, 11 | }; 12 | 13 | export default VoyageConfig; 14 | -------------------------------------------------------------------------------- /src/providers/workers-ai/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const WorkersAiAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: ({ providerOptions }) => { 5 | const { workersAiAccountId } = providerOptions; 6 | return `https://api.cloudflare.com/client/v4/accounts/${workersAiAccountId}/ai/run`; 7 | }, 8 | headers: ({ providerOptions }) => { 9 | const { apiKey } = providerOptions; 10 | return { Authorization: `Bearer ${apiKey}` }; 11 | }, 12 | getEndpoint: ({ fn, gatewayRequestBodyJSON: params }) => { 13 | const { model } = params; 14 | switch (fn) { 15 | case 'complete': { 16 | return `/${model}`; 17 | } 18 | case 'chatComplete': { 19 | return `/${model}`; 20 | } 21 | case 'embed': { 22 | return `/${model}`; 23 | } 24 | case 'imageGenerate': { 25 | return `/${model}`; 26 | } 27 | default: 28 | return ''; 29 | } 30 | }, 31 | }; 32 | 33 | export default WorkersAiAPIConfig; 34 | -------------------------------------------------------------------------------- /src/providers/workers-ai/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import WorkersAiAPIConfig from './api'; 3 | import { 4 | WorkersAiChatCompleteConfig, 5 | WorkersAiChatCompleteResponseTransform, 6 | WorkersAiChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | import { 9 | WorkersAiCompleteConfig, 10 | WorkersAiCompleteResponseTransform, 11 | WorkersAiCompleteStreamChunkTransform, 12 | } from './complete'; 13 | import { WorkersAiEmbedConfig, WorkersAiEmbedResponseTransform } from './embed'; 14 | import { 15 | WorkersAiImageGenerateConfig, 16 | WorkersAiImageGenerateResponseTransform, 17 | } from './imageGenerate'; 18 | 19 | const WorkersAiConfig: ProviderConfigs = { 20 | complete: WorkersAiCompleteConfig, 21 | chatComplete: WorkersAiChatCompleteConfig, 22 | api: WorkersAiAPIConfig, 23 | embed: WorkersAiEmbedConfig, 24 | imageGenerate: WorkersAiImageGenerateConfig, 25 | responseTransforms: { 26 | 'stream-complete': WorkersAiCompleteStreamChunkTransform, 27 | complete: WorkersAiCompleteResponseTransform, 28 | chatComplete: WorkersAiChatCompleteResponseTransform, 29 | 'stream-chatComplete': WorkersAiChatCompleteStreamChunkTransform, 30 | embed: WorkersAiEmbedResponseTransform, 31 | imageGenerate: WorkersAiImageGenerateResponseTransform, 32 | }, 33 | }; 34 | 35 | export default WorkersAiConfig; 36 | -------------------------------------------------------------------------------- /src/providers/workers-ai/utils.ts: -------------------------------------------------------------------------------- 1 | import { ErrorResponse } from '../types'; 2 | import { generateErrorResponse } from '../utils'; 3 | import { WORKERS_AI } from '../../globals'; 4 | 5 | export interface WorkersAiErrorResponse { 6 | success: boolean; 7 | errors: WorkersAiErrorObject[]; 8 | } 9 | 10 | export interface WorkersAiErrorObject { 11 | code: string; 12 | message: string; 13 | } 14 | 15 | export const WorkersAiErrorResponseTransform: ( 16 | response: WorkersAiErrorResponse 17 | ) => ErrorResponse | undefined = (response) => { 18 | if ('errors' in response) { 19 | return generateErrorResponse( 20 | { 21 | message: response.errors 22 | ?.map((error) => `Error ${error.code}:${error.message}`) 23 | .join(', '), 24 | type: null, 25 | param: null, 26 | code: null, 27 | }, 28 | WORKERS_AI 29 | ); 30 | } 31 | 32 | return undefined; 33 | }; 34 | -------------------------------------------------------------------------------- /src/providers/x-ai/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const XAIAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://api.x.ai/v1', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/chat/completions'; 12 | case 'complete': 13 | return '/completions'; 14 | case 'embed': 15 | return '/embeddings'; 16 | default: 17 | return ''; 18 | } 19 | }, 20 | }; 21 | 22 | export default XAIAPIConfig; 23 | -------------------------------------------------------------------------------- /src/providers/x-ai/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import { X_AI } from '../../globals'; 3 | import XAIAPIConfig from './api'; 4 | import { 5 | chatCompleteParams, 6 | completeParams, 7 | embedParams, 8 | responseTransformers, 9 | } from '../open-ai-base'; 10 | 11 | const XAIConfig: ProviderConfigs = { 12 | chatComplete: chatCompleteParams([], { model: 'grok-beta' }), 13 | complete: completeParams([], { model: 'grok-beta' }), 14 | embed: embedParams([], { model: 'v1' }), 15 | api: XAIAPIConfig, 16 | responseTransforms: responseTransformers(X_AI, { 17 | chatComplete: true, 18 | complete: true, 19 | embed: true, 20 | }), 21 | }; 22 | 23 | export default XAIConfig; 24 | -------------------------------------------------------------------------------- /src/providers/zhipu/api.ts: -------------------------------------------------------------------------------- 1 | import { ProviderAPIConfig } from '../types'; 2 | 3 | const ZhipuAPIConfig: ProviderAPIConfig = { 4 | getBaseURL: () => 'https://open.bigmodel.cn/api/paas/v4', 5 | headers: ({ providerOptions }) => { 6 | return { Authorization: `Bearer ${providerOptions.apiKey}` }; // https://open.bigmodel.cn/usercenter/apikeys 7 | }, 8 | getEndpoint: ({ fn }) => { 9 | switch (fn) { 10 | case 'chatComplete': 11 | return '/chat/completions'; 12 | case 'embed': 13 | return '/embeddings'; 14 | default: 15 | return ''; 16 | } 17 | }, 18 | }; 19 | 20 | export default ZhipuAPIConfig; 21 | -------------------------------------------------------------------------------- /src/providers/zhipu/embed.ts: -------------------------------------------------------------------------------- 1 | import { ZHIPU } from '../../globals'; 2 | import { EmbedResponse } from '../../types/embedRequestBody'; 3 | import { ErrorResponse, ProviderConfig } from '../types'; 4 | import { generateErrorResponse } from '../utils'; 5 | 6 | export const ZhipuEmbedConfig: ProviderConfig = { 7 | model: { 8 | param: 'model', 9 | required: true, 10 | default: 'embedding-2', 11 | }, 12 | input: { 13 | param: 'input', 14 | required: true, 15 | }, 16 | }; 17 | 18 | interface ZhipuEmbedResponse extends EmbedResponse {} 19 | 20 | export const ZhipuEmbedResponseTransform: ( 21 | response: ZhipuEmbedResponse | ErrorResponse, 22 | responseStatus: number 23 | ) => EmbedResponse | ErrorResponse = (response, responseStatus) => { 24 | if (responseStatus !== 200 && 'error' in response) { 25 | return generateErrorResponse( 26 | { 27 | message: response.error.message, 28 | type: response.error.type, 29 | param: response.error.param, 30 | code: response.error.code, 31 | }, 32 | ZHIPU 33 | ); 34 | } 35 | 36 | return response; 37 | }; 38 | -------------------------------------------------------------------------------- /src/providers/zhipu/index.ts: -------------------------------------------------------------------------------- 1 | import { ProviderConfigs } from '../types'; 2 | import ZhipuAPIConfig from './api'; 3 | import { 4 | ZhipuChatCompleteConfig, 5 | ZhipuChatCompleteResponseTransform, 6 | ZhipuChatCompleteStreamChunkTransform, 7 | } from './chatComplete'; 8 | import { ZhipuEmbedConfig, ZhipuEmbedResponseTransform } from './embed'; 9 | 10 | const ZhipuConfig: ProviderConfigs = { 11 | chatComplete: ZhipuChatCompleteConfig, 12 | embed: ZhipuEmbedConfig, 13 | api: ZhipuAPIConfig, 14 | responseTransforms: { 15 | chatComplete: ZhipuChatCompleteResponseTransform, 16 | 'stream-chatComplete': ZhipuChatCompleteStreamChunkTransform, 17 | embed: ZhipuEmbedResponseTransform, 18 | }, 19 | }; 20 | 21 | export default ZhipuConfig; 22 | -------------------------------------------------------------------------------- /src/tests/common.test.ts: -------------------------------------------------------------------------------- 1 | import Providers from '../providers'; 2 | import testVariables from './resources/testVariables'; 3 | import { executeChatCompletionEndpointTests } from './routeSpecificTestFunctions.ts/chatCompletion'; 4 | 5 | for (const provider in testVariables) { 6 | const variables = testVariables[provider]; 7 | const config = Providers[provider]; 8 | 9 | if (!variables.apiKey) { 10 | console.log(`Skipping ${provider} as API key is not provided`); 11 | continue; 12 | } 13 | 14 | if (config.chatComplete) { 15 | describe(`${provider} /chat/completions endpoint tests:`, () => 16 | executeChatCompletionEndpointTests(provider, variables)); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/tests/resources/constants.ts: -------------------------------------------------------------------------------- 1 | const baseURL = 'http://localhost'; 2 | export const CHAT_COMPLETIONS_ENDPOINT = `${baseURL}/v1/chat/completions`; 3 | -------------------------------------------------------------------------------- /src/tests/resources/requestTemplates.ts: -------------------------------------------------------------------------------- 1 | import { Params } from '../../types/requestBody'; 2 | 3 | const CHAT_COMPLETE_WITH_MESSAGE_CONTENT_ARRAYS_REQUEST: Params = { 4 | model: 'MODEL_PLACE_HOLDER', 5 | max_tokens: 20, 6 | stream: false, 7 | messages: [ 8 | { 9 | role: 'system', 10 | content: 'You are the half-blood prince', 11 | }, 12 | { 13 | role: 'user', 14 | content: [ 15 | { 16 | type: 'text', 17 | text: 'Can you teach me a useful spell?', 18 | }, 19 | ], 20 | }, 21 | ], 22 | }; 23 | 24 | export const getChatCompleteWithMessageContentArraysRequest = ( 25 | model?: string 26 | ) => { 27 | return JSON.stringify({ 28 | ...CHAT_COMPLETE_WITH_MESSAGE_CONTENT_ARRAYS_REQUEST, 29 | model, 30 | }); 31 | }; 32 | 33 | export const CHAT_COMPLETE_WITH_MESSAGE_STRING_REQUEST: Params = { 34 | model: 'MODEL_PLACEHOLDER', 35 | max_tokens: 20, 36 | stream: false, 37 | messages: [ 38 | { 39 | role: 'system', 40 | content: 'You are the half-blood prince', 41 | }, 42 | { 43 | role: 'user', 44 | content: 'Can you teach me a useful spell?', 45 | }, 46 | ], 47 | }; 48 | 49 | export const getChatCompleteWithMessageStringRequest = (model?: string) => { 50 | return JSON.stringify({ 51 | ...CHAT_COMPLETE_WITH_MESSAGE_STRING_REQUEST, 52 | model, 53 | }); 54 | }; 55 | -------------------------------------------------------------------------------- /src/tests/resources/utils.ts: -------------------------------------------------------------------------------- 1 | export const createDefaultHeaders = ( 2 | provider: string, 3 | authorization: string 4 | ) => { 5 | return { 6 | 'x-portkey-provider': provider, 7 | Authorization: authorization, 8 | 'Content-Type': 'application/json', 9 | }; 10 | }; 11 | -------------------------------------------------------------------------------- /src/types/responseBody.ts: -------------------------------------------------------------------------------- 1 | interface CitationSource { 2 | startIndex?: number; 3 | endIndex?: number; 4 | uri?: string; 5 | license?: string; 6 | } 7 | 8 | interface CitationMetadata { 9 | citationSources?: CitationSource[]; 10 | } 11 | 12 | interface PalmMessage { 13 | content?: string; 14 | author?: string; 15 | citationMetadata?: CitationMetadata; 16 | } 17 | 18 | interface ContentFilter { 19 | reason: 'BLOCKED_REASON_UNSPECIFIED' | 'SAFETY' | 'OTHER'; 20 | message: string; 21 | } 22 | 23 | export interface PalmChatCompleteResponse { 24 | candidates: PalmMessage[]; 25 | messages: PalmMessage[]; 26 | filters: ContentFilter[]; 27 | error?: PalmError; 28 | } 29 | 30 | interface PalmTextOutput { 31 | output: string; 32 | safetyRatings: safetyRatings[]; 33 | } 34 | 35 | interface safetyRatings { 36 | category: 37 | | 'HARM_CATEGORY_DEROGATORY' 38 | | 'HARM_CATEGORY_TOXICITY' 39 | | 'HARM_CATEGORY_VIOLENCE' 40 | | 'HARM_CATEGORY_SEXUAL' 41 | | 'HARM_CATEGORY_MEDICAL' 42 | | 'HARM_CATEGORY_DANGEROUS'; 43 | probability: 'NEGLIGIBLE' | 'LOW' | 'HIGH'; 44 | } 45 | 46 | interface PalmFilter { 47 | reason: 'OTHER'; 48 | } 49 | 50 | interface PalmError { 51 | code: number; 52 | message: string; 53 | status: string; 54 | } 55 | 56 | export interface PalmCompleteResponse { 57 | candidates: PalmTextOutput[]; 58 | filters: PalmFilter[]; 59 | error?: PalmError; 60 | } 61 | -------------------------------------------------------------------------------- /src/utils/misc.ts: -------------------------------------------------------------------------------- 1 | export function toSnakeCase(str: string) { 2 | return str 3 | .replace(/([a-z])([A-Z])/g, '$1_$2') // Handle camelCase and PascalCase 4 | .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2') // Handle acronyms 5 | .replace(/[^a-zA-Z0-9]+/g, '_') // Replace special characters with '_' 6 | .replace(/_+/g, '_') // Merge multiple underscores 7 | .toLowerCase(); 8 | } 9 | -------------------------------------------------------------------------------- /start-test.js: -------------------------------------------------------------------------------- 1 | import { spawn } from 'node:child_process'; 2 | 3 | console.log('Starting the application...'); 4 | 5 | const app = spawn('node', ['build/start-server.js', '--headless'], { 6 | stdio: 'inherit', 7 | }); 8 | 9 | // Listen for errors when spawning the process 10 | app.on('exit', (err) => { 11 | console.error('Failed to start the app:', err); 12 | process.exit(1); // Exit with a failure code if there is an error 13 | }); 14 | 15 | // Listen for when the process starts 16 | app.on('spawn', () => { 17 | console.log('App started successfully'); 18 | setTimeout(() => { 19 | app.kill(); 20 | process.exit(0); 21 | }, 3000); 22 | }); 23 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "strict": true, 4 | "target": "ESNext", 5 | "module": "ESNext", 6 | "moduleResolution": "node", 7 | "esModuleInterop": true, 8 | "skipLibCheck": true, 9 | "lib": ["esnext"], 10 | "types": ["@cloudflare/workers-types", "node", "jest"], 11 | "jsx": "react-jsx", 12 | "jsxImportSource": "hono/jsx", 13 | "resolveJsonModule": true 14 | }, 15 | "exclude": ["**/*.test.ts", "cookbook"] 16 | } 17 | -------------------------------------------------------------------------------- /wrangler.toml: -------------------------------------------------------------------------------- 1 | name = "rubeus" 2 | compatibility_date = "2024-12-05" 3 | main = "src/index.ts" 4 | compatibility_flags = [ "nodejs_compat" ] 5 | 6 | [vars] 7 | ENVIRONMENT = 'dev' 8 | CUSTOM_HEADERS_TO_IGNORE = [] 9 | 10 | # 11 | #Configuration for DEVELOPMENT environment 12 | # 13 | [env.staging] 14 | name = "rubeus-dev" 15 | 16 | [env.staging.vars] 17 | ENVIRONMENT = 'staging' 18 | CUSTOM_HEADERS_TO_IGNORE = [] 19 | 20 | 21 | # 22 | #Configuration for PRODUCTION environment 23 | # 24 | [env.production] 25 | name = "rubeus" 26 | logpush=true 27 | 28 | [env.production.vars] 29 | ENVIRONMENT = 'production' 30 | CUSTOM_HEADERS_TO_IGNORE = [] 31 | --------------------------------------------------------------------------------