├── public └── empty.html ├── .gitignore ├── vercel.json ├── netlify.toml ├── wrangler.toml ├── netlify ├── functions │ └── handler.mjs └── edge-functions │ └── handler.mjs ├── deno.mjs ├── .editorconfig ├── bun.mjs ├── node.mjs ├── package.json ├── .github └── workflows │ └── cf-deploy.yml ├── api └── handler.mjs ├── LICENSE ├── readme.MD └── src └── worker.mjs /public/empty.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .netlify 3 | .vercel 4 | .wrangler 5 | -------------------------------------------------------------------------------- /vercel.json: -------------------------------------------------------------------------------- 1 | { 2 | "rewrites": [ 3 | { "source": "/(.*)", "destination": "api/handler" } 4 | ] 5 | } -------------------------------------------------------------------------------- /netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | publish = "public/" 3 | 4 | [functions] 5 | node_bundler = "esbuild" -------------------------------------------------------------------------------- /wrangler.toml: -------------------------------------------------------------------------------- 1 | name = "gemini" 2 | main = "src/worker.mjs" 3 | compatibility_date = "2024-09-23" 4 | compatibility_flags = [ "nodejs_compat" ] 5 | -------------------------------------------------------------------------------- /netlify/functions/handler.mjs: -------------------------------------------------------------------------------- 1 | export const config = { path: "/*" }; 2 | 3 | import worker from "../../src/worker.mjs"; 4 | 5 | export default worker.fetch; 6 | -------------------------------------------------------------------------------- /netlify/edge-functions/handler.mjs: -------------------------------------------------------------------------------- 1 | export const config = { path: "/edge/*" }; 2 | 3 | import worker from "../../src/worker.mjs"; 4 | 5 | export default worker.fetch; 6 | -------------------------------------------------------------------------------- /deno.mjs: -------------------------------------------------------------------------------- 1 | //deprecated: 2 | //import {serve} from "https://deno.land/std/http/mod.ts" 3 | 4 | import worker from "./src/worker.mjs"; 5 | 6 | const port = +(Deno.env.get("PORT") ?? 8080); 7 | 8 | Deno.serve({port}, worker.fetch); 9 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.yml] 13 | indent_style = space 14 | -------------------------------------------------------------------------------- /bun.mjs: -------------------------------------------------------------------------------- 1 | import worker from "./src/worker.mjs"; 2 | 3 | worker.port = +(process.env.PORT || 8080); 4 | const environment = process.env.NODE_ENV ?? "development" 5 | worker.development = process.env.NODE_ENV==="development" 6 | 7 | const server = Bun.serve(worker); 8 | console.log(`[${environment}] Listening on ${server.url.origin}`); 9 | -------------------------------------------------------------------------------- /node.mjs: -------------------------------------------------------------------------------- 1 | import { createServerAdapter } from '@whatwg-node/server' 2 | import { createServer } from 'node:http' 3 | import worker from "./src/worker.mjs"; 4 | 5 | const port = +(process.env.PORT || 8080); 6 | 7 | const serverAdapter = createServerAdapter(worker.fetch) 8 | const server = createServer(serverAdapter) 9 | server.listen(port, () => { 10 | console.log('Listening on:', server.address()); 11 | }) 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "@whatwg-node/server": "0.9" 4 | }, 5 | "devDependencies": { 6 | "nodemon": "3" 7 | }, 8 | "scripts": { 9 | "start": "node node.mjs", 10 | "dev": "nodemon --watch node.mjs --watch src/*.mjs node.mjs", 11 | "start:deno": "deno --allow-env --allow-net deno.mjs", 12 | "dev:deno": "deno --watch --allow-env --allow-net deno.mjs", 13 | "start:bun": "bun bun.mjs", 14 | "dev:bun": "bun --watch bun.mjs" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.github/workflows/cf-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Worker 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | repository_dispatch: 8 | jobs: 9 | deploy: 10 | runs-on: ubuntu-latest 11 | name: Deploy 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Build & Deploy Worker 15 | uses: cloudflare/wrangler-action@v3 16 | with: 17 | apiToken: ${{ secrets.CF_API_TOKEN }} 18 | accountId: ${{ secrets.CF_ACCOUNT_ID }} 19 | -------------------------------------------------------------------------------- /api/handler.mjs: -------------------------------------------------------------------------------- 1 | import worker from "../src/worker.mjs"; 2 | 3 | export default worker.fetch; 4 | 5 | export const config = { 6 | runtime: "edge", 7 | // Available languages and regions for Google AI Studio and Gemini API 8 | // https://ai.google.dev/gemini-api/docs/available-regions#available_regions 9 | // Vercel Edge Network Regions 10 | // https://vercel.com/docs/edge-network/regions#region-list 11 | regions: [ 12 | "arn1", 13 | "bom1", 14 | "cdg1", 15 | "cle1", 16 | "cpt1", 17 | "dub1", 18 | "fra1", 19 | "gru1", 20 | //"hkg1", 21 | "hnd1", 22 | "iad1", 23 | "icn1", 24 | "kix1", 25 | "pdx1", 26 | "sfo1", 27 | "sin1", 28 | "syd1", 29 | ], 30 | }; 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 PublicAffairs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /readme.MD: -------------------------------------------------------------------------------- 1 | ## Why 2 | 3 | The Gemini API is [free](https://ai.google.dev/pricing "limits applied!"), 4 | but there are many tools that work exclusively with the OpenAI API. 5 | 6 | This project provides a personal OpenAI-compatible endpoint for free. 7 | 8 | 9 | ## Serverless? 10 | 11 | Although it runs in the cloud, it does not require server maintenance. 12 | It can be easily deployed to various providers for free 13 | (with generous limits suitable for personal use). 14 | 15 | > [!TIP] 16 | > Running the proxy endpoint locally is also an option, 17 | > though it's more appropriate for development use. 18 | 19 | 20 | ## How to start 21 | 22 | You will need a personal Google [API key](https://makersuite.google.com/app/apikey). 23 | 24 | > [!IMPORTANT] 25 | > Even if you are located outside of the [supported regions](https://ai.google.dev/gemini-api/docs/available-regions#available_regions), 26 | > it is still possible to acquire one using a VPN. 27 | 28 | Deploy the project to one of the providers, using the instructions below. 29 | You will need to set up an account there. 30 | 31 | If you opt for “button-deploy”, you'll be guided through the process of forking the repository first, 32 | which is necessary for continuous integration (CI). 33 | 34 | 35 | ### Deploy with Vercel 36 | 37 | [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https://github.com/PublicAffairs/openai-gemini&repository-name=my-openai-gemini) 38 | - Alternatively can be deployed with [cli](https://vercel.com/docs/cli): 39 | `vercel deploy` 40 | - Serve locally: `vercel dev` 41 | - Vercel _Functions_ [limitations](https://vercel.com/docs/functions/limitations) (with _Edge_ runtime) 42 | 43 | 44 | ### Deploy to Netlify 45 | 46 | [![Deploy to Netlify](https://www.netlify.com/img/deploy/button.svg)](https://app.netlify.com/start/deploy?repository=https://github.com/PublicAffairs/openai-gemini&integrationName=integrationName&integrationSlug=integrationSlug&integrationDescription=integrationDescription) 47 | - Alternatively can be deployed with [cli](https://docs.netlify.com/cli/get-started/): 48 | `netlify deploy` 49 | - Serve locally: `netlify dev` 50 | - Two different api bases provided: 51 | - `/v1` (e.g. `/v1/chat/completions` endpoint) 52 | _Functions_ [limits](https://docs.netlify.com/functions/get-started/?fn-language=js#synchronous-function-2) 53 | - `/edge/v1` 54 | _Edge functions_ [limits](https://docs.netlify.com/edge-functions/limits/) 55 | 56 | 57 | ### Deploy to Cloudflare 58 | 59 | [![Deploy to Cloudflare Workers](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https://github.com/PublicAffairs/openai-gemini) 60 | - Alternatively can be deployed manually pasting content of [`src/worker.mjs`](src/worker.mjs) 61 | to https://workers.cloudflare.com/playground (see there `Deploy` button). 62 | - Alternatively can be deployed with [cli](https://developers.cloudflare.com/workers/wrangler/): 63 | `wrangler deploy` 64 | - Serve locally: `wrangler dev` 65 | - _Worker_ [limits](https://developers.cloudflare.com/workers/platform/limits/#worker-limits) 66 | 67 | 68 | ### Deploy to Deno 69 | 70 | See details [here](https://github.com/PublicAffairs/openai-gemini/discussions/19). 71 | 72 | 73 | ### Serve locally - with Node, Deno, Bun 74 | 75 | Only for Node: `npm install`. 76 | 77 | Then `npm run start` / `npm run start:deno` / `npm run start:bun`. 78 | 79 | 80 | #### Dev mode (watch source changes) 81 | 82 | Only for Node: `npm install --include=dev` 83 | 84 | Then: `npm run dev` / `npm run dev:deno` / `npm run dev:bun`. 85 | 86 | 87 | ## How to use 88 | If you open your newly-deployed site in a browser, you will only see a `404 Not Found` message. This is expected, as the API is not designed for direct browser access. 89 | To utilize it, you should enter your API address and your Gemini API key into the corresponding fields in your software settings. 90 | 91 | > [!NOTE] 92 | > Not all software tools allow overriding the OpenAI endpoint, but many do 93 | > (however these settings can sometimes be deeply hidden). 94 | 95 | Typically, you should specify the API base in this format: 96 | `https://my-super-proxy.vercel.app/v1` 97 | 98 | The relevant field may be labeled as "_OpenAI proxy_". 99 | You might need to look under "_Advanced settings_" or similar sections. 100 | Alternatively, it could be in some config file (check the relevant documentation for details). 101 | 102 | For some command-line tools, you may need to set an environment variable, _e.g._: 103 | ```sh 104 | OPENAI_BASE_URL="https://my-super-proxy.vercel.app/v1" 105 | ``` 106 | _..or_: 107 | ```sh 108 | OPENAI_API_BASE="https://my-super-proxy.vercel.app/v1" 109 | ``` 110 | 111 | 112 | ## Models 113 | 114 | Requests use the specified [model] if its name starts with "gemini-", "learnlm-", 115 | or "models/". Otherwise, these defaults apply: 116 | 117 | - `chat/completions`: `gemini-1.5-pro-latest` 118 | - `embeddings`: `text-embedding-004` 119 | 120 | [model]: https://ai.google.dev/gemini-api/docs/models/gemini 121 | 122 | 123 | ## Media 124 | 125 | [Vision] and [audio] input supported as per OpenAI [specs]. 126 | Implemented via [`inlineData`](https://ai.google.dev/api/caching#Part). 127 | 128 | [vision]: https://platform.openai.com/docs/guides/vision 129 | [audio]: https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in 130 | [specs]: https://platform.openai.com/docs/api-reference/chat/create 131 | 132 | --- 133 | 134 | ## Supported API endpoints and applicable parameters 135 | 136 | - [x] `chat/completions` 137 | 138 | Currently, most of the parameters that are applicable to both APIs have been implemented, 139 | with the exception of function calls. 140 |
141 | 142 | - [x] `messages` 143 | - [x] `content` 144 | - [x] `role` 145 | - [x] "system" (=>`system_instruction`) 146 | - [x] "user" 147 | - [x] "assistant" 148 | - [ ] "tool" (v1beta) 149 | - [ ] `name` 150 | - [ ] `tool_calls` 151 | - [x] `model` 152 | - [x] `frequency_penalty` 153 | - [ ] `logit_bias` 154 | - [ ] `logprobs` 155 | - [ ] `top_logprobs` 156 | - [x] `max_tokens`, `max_completion_tokens` 157 | - [x] `n` (`candidateCount` <8, not for streaming) 158 | - [x] `presence_penalty` 159 | - [x] `response_format` 160 | - [x] "json_object" 161 | - [x] "json_schema" (a select subset of an OpenAPI 3.0 schema object) 162 | - [x] "text" 163 | - [ ] `seed` 164 | - [x] `stop`: string|array (`stopSequences` [1,5]) 165 | - [x] `stream` 166 | - [x] `stream_options` 167 | - [x] `include_usage` 168 | - [x] `temperature` (0.0..2.0 for OpenAI, but Gemini supports up to infinity) 169 | - [x] `top_p` 170 | - [ ] `tools` (v1beta) 171 | - [ ] `tool_choice` (v1beta) 172 | - [ ] `parallel_tool_calls` 173 | 174 |
175 | - [ ] `completions` 176 | - [x] `embeddings` 177 | - [x] `models` 178 | -------------------------------------------------------------------------------- /src/worker.mjs: -------------------------------------------------------------------------------- 1 | import { Buffer } from "node:buffer"; 2 | 3 | export default { 4 | async fetch (request) { 5 | if (request.method === "OPTIONS") { 6 | return handleOPTIONS(); 7 | } 8 | const errHandler = (err) => { 9 | console.error(err); 10 | return new Response(err.message, fixCors({ status: err.status ?? 500 })); 11 | }; 12 | try { 13 | const auth = request.headers.get("Authorization"); 14 | const apiKey = auth?.split(" ")[1]; 15 | const assert = (success) => { 16 | if (!success) { 17 | throw new HttpError("The specified HTTP method is not allowed for the requested resource", 400); 18 | } 19 | }; 20 | const { pathname } = new URL(request.url); 21 | switch (true) { 22 | case pathname.endsWith("/chat/completions"): 23 | assert(request.method === "POST"); 24 | return handleCompletions(await request.json(), apiKey) 25 | .catch(errHandler); 26 | case pathname.endsWith("/embeddings"): 27 | assert(request.method === "POST"); 28 | return handleEmbeddings(await request.json(), apiKey) 29 | .catch(errHandler); 30 | case pathname.endsWith("/models"): 31 | assert(request.method === "GET"); 32 | return handleModels(apiKey) 33 | .catch(errHandler); 34 | default: 35 | throw new HttpError("404 Not Found", 404); 36 | } 37 | } catch (err) { 38 | return errHandler(err); 39 | } 40 | } 41 | }; 42 | 43 | class HttpError extends Error { 44 | constructor(message, status) { 45 | super(message); 46 | this.name = this.constructor.name; 47 | this.status = status; 48 | } 49 | } 50 | 51 | const fixCors = ({ headers, status, statusText }) => { 52 | headers = new Headers(headers); 53 | headers.set("Access-Control-Allow-Origin", "*"); 54 | return { headers, status, statusText }; 55 | }; 56 | 57 | const handleOPTIONS = async () => { 58 | return new Response(null, { 59 | headers: { 60 | "Access-Control-Allow-Origin": "*", 61 | "Access-Control-Allow-Methods": "*", 62 | "Access-Control-Allow-Headers": "*", 63 | } 64 | }); 65 | }; 66 | 67 | const BASE_URL = "https://generativelanguage.googleapis.com"; 68 | const API_VERSION = "v1beta"; 69 | 70 | // https://github.com/google-gemini/generative-ai-js/blob/cf223ff4a1ee5a2d944c53cddb8976136382bee6/src/requests/request.ts#L71 71 | const API_CLIENT = "genai-js/0.21.0"; // npm view @google/generative-ai version 72 | const makeHeaders = (apiKey, more) => ({ 73 | "x-goog-api-client": API_CLIENT, 74 | ...(apiKey && { "x-goog-api-key": apiKey }), 75 | ...more 76 | }); 77 | 78 | async function handleModels (apiKey) { 79 | const response = await fetch(`${BASE_URL}/${API_VERSION}/models`, { 80 | headers: makeHeaders(apiKey), 81 | }); 82 | let { body } = response; 83 | if (response.ok) { 84 | const { models } = JSON.parse(await response.text()); 85 | body = JSON.stringify({ 86 | object: "list", 87 | data: models.map(({ name }) => ({ 88 | id: name.replace("models/", ""), 89 | object: "model", 90 | created: 0, 91 | owned_by: "", 92 | })), 93 | }, null, " "); 94 | } 95 | return new Response(body, fixCors(response)); 96 | } 97 | 98 | const DEFAULT_EMBEDDINGS_MODEL = "text-embedding-004"; 99 | async function handleEmbeddings (req, apiKey) { 100 | if (typeof req.model !== "string") { 101 | throw new HttpError("model is not specified", 400); 102 | } 103 | if (!Array.isArray(req.input)) { 104 | req.input = [ req.input ]; 105 | } 106 | let model; 107 | if (req.model.startsWith("models/")) { 108 | model = req.model; 109 | } else { 110 | req.model = DEFAULT_EMBEDDINGS_MODEL; 111 | model = "models/" + req.model; 112 | } 113 | const response = await fetch(`${BASE_URL}/${API_VERSION}/${model}:batchEmbedContents`, { 114 | method: "POST", 115 | headers: makeHeaders(apiKey, { "Content-Type": "application/json" }), 116 | body: JSON.stringify({ 117 | "requests": req.input.map(text => ({ 118 | model, 119 | content: { parts: { text } }, 120 | outputDimensionality: req.dimensions, 121 | })) 122 | }) 123 | }); 124 | let { body } = response; 125 | if (response.ok) { 126 | const { embeddings } = JSON.parse(await response.text()); 127 | body = JSON.stringify({ 128 | object: "list", 129 | data: embeddings.map(({ values }, index) => ({ 130 | object: "embedding", 131 | index, 132 | embedding: values, 133 | })), 134 | model: req.model, 135 | }, null, " "); 136 | } 137 | return new Response(body, fixCors(response)); 138 | } 139 | 140 | const DEFAULT_MODEL = "gemini-1.5-pro-latest"; 141 | async function handleCompletions (req, apiKey) { 142 | let model = DEFAULT_MODEL; 143 | switch(true) { 144 | case typeof req.model !== "string": 145 | break; 146 | case req.model.startsWith("models/"): 147 | model = req.model.substring(7); 148 | break; 149 | case req.model.startsWith("gemini-"): 150 | case req.model.startsWith("learnlm-"): 151 | model = req.model; 152 | } 153 | const TASK = req.stream ? "streamGenerateContent" : "generateContent"; 154 | let url = `${BASE_URL}/${API_VERSION}/models/${model}:${TASK}`; 155 | if (req.stream) { url += "?alt=sse"; } 156 | const response = await fetch(url, { 157 | method: "POST", 158 | headers: makeHeaders(apiKey, { "Content-Type": "application/json" }), 159 | body: JSON.stringify(await transformRequest(req)), // try 160 | }); 161 | 162 | let body = response.body; 163 | if (response.ok) { 164 | let id = generateChatcmplId(); //"chatcmpl-8pMMaqXMK68B3nyDBrapTDrhkHBQK"; 165 | if (req.stream) { 166 | body = response.body 167 | .pipeThrough(new TextDecoderStream()) 168 | .pipeThrough(new TransformStream({ 169 | transform: parseStream, 170 | flush: parseStreamFlush, 171 | buffer: "", 172 | })) 173 | .pipeThrough(new TransformStream({ 174 | transform: toOpenAiStream, 175 | flush: toOpenAiStreamFlush, 176 | streamIncludeUsage: req.stream_options?.include_usage, 177 | model, id, last: [], 178 | })) 179 | .pipeThrough(new TextEncoderStream()); 180 | } else { 181 | body = await response.text(); 182 | body = processCompletionsResponse(JSON.parse(body), model, id); 183 | } 184 | } 185 | return new Response(body, fixCors(response)); 186 | } 187 | 188 | const harmCategory = [ 189 | "HARM_CATEGORY_HATE_SPEECH", 190 | "HARM_CATEGORY_SEXUALLY_EXPLICIT", 191 | "HARM_CATEGORY_DANGEROUS_CONTENT", 192 | "HARM_CATEGORY_HARASSMENT", 193 | "HARM_CATEGORY_CIVIC_INTEGRITY", 194 | ]; 195 | const safetySettings = harmCategory.map(category => ({ 196 | category, 197 | threshold: "BLOCK_NONE", 198 | })); 199 | const fieldsMap = { 200 | stop: "stopSequences", 201 | n: "candidateCount", // not for streaming 202 | max_tokens: "maxOutputTokens", 203 | max_completion_tokens: "maxOutputTokens", 204 | temperature: "temperature", 205 | top_p: "topP", 206 | top_k: "topK", // non-standard 207 | frequency_penalty: "frequencyPenalty", 208 | presence_penalty: "presencePenalty", 209 | }; 210 | const transformConfig = (req) => { 211 | let cfg = {}; 212 | //if (typeof req.stop === "string") { req.stop = [req.stop]; } // no need 213 | for (let key in req) { 214 | const matchedKey = fieldsMap[key]; 215 | if (matchedKey) { 216 | cfg[matchedKey] = req[key]; 217 | } 218 | } 219 | if (req.response_format) { 220 | switch(req.response_format.type) { 221 | case "json_schema": 222 | cfg.responseSchema = req.response_format.json_schema?.schema; 223 | if (cfg.responseSchema && "enum" in cfg.responseSchema) { 224 | cfg.responseMimeType = "text/x.enum"; 225 | break; 226 | } 227 | // eslint-disable-next-line no-fallthrough 228 | case "json_object": 229 | cfg.responseMimeType = "application/json"; 230 | break; 231 | case "text": 232 | cfg.responseMimeType = "text/plain"; 233 | break; 234 | default: 235 | throw new HttpError("Unsupported response_format.type", 400); 236 | } 237 | } 238 | return cfg; 239 | }; 240 | 241 | const parseImg = async (url) => { 242 | let mimeType, data; 243 | if (url.startsWith("http://") || url.startsWith("https://")) { 244 | try { 245 | const response = await fetch(url); 246 | if (!response.ok) { 247 | throw new Error(`${response.status} ${response.statusText} (${url})`); 248 | } 249 | mimeType = response.headers.get("content-type"); 250 | data = Buffer.from(await response.arrayBuffer()).toString("base64"); 251 | } catch (err) { 252 | throw new Error("Error fetching image: " + err.toString()); 253 | } 254 | } else { 255 | const match = url.match(/^data:(?.*?)(;base64)?,(?.*)$/); 256 | if (!match) { 257 | throw new Error("Invalid image data: " + url); 258 | } 259 | ({ mimeType, data } = match.groups); 260 | } 261 | return { 262 | inlineData: { 263 | mimeType, 264 | data, 265 | }, 266 | }; 267 | }; 268 | 269 | const transformMsg = async ({ role, content }) => { 270 | const parts = []; 271 | if (!Array.isArray(content)) { 272 | // system, user: string 273 | // assistant: string or null (Required unless tool_calls is specified.) 274 | parts.push({ text: content }); 275 | return { role, parts }; 276 | } 277 | // user: 278 | // An array of content parts with a defined type. 279 | // Supported options differ based on the model being used to generate the response. 280 | // Can contain text, image, or audio inputs. 281 | for (const item of content) { 282 | switch (item.type) { 283 | case "text": 284 | parts.push({ text: item.text }); 285 | break; 286 | case "image_url": 287 | parts.push(await parseImg(item.image_url.url)); 288 | break; 289 | case "input_audio": 290 | parts.push({ 291 | inlineData: { 292 | mimeType: "audio/" + item.input_audio.format, 293 | data: item.input_audio.data, 294 | } 295 | }); 296 | break; 297 | default: 298 | throw new TypeError(`Unknown "content" item type: "${item.type}"`); 299 | } 300 | } 301 | if (content.every(item => item.type === "image_url")) { 302 | parts.push({ text: "" }); // to avoid "Unable to submit request because it must have a text parameter" 303 | } 304 | return { role, parts }; 305 | }; 306 | 307 | const transformMessages = async (messages) => { 308 | if (!messages) { return; } 309 | const contents = []; 310 | let system_instruction; 311 | for (const item of messages) { 312 | if (item.role === "system") { 313 | delete item.role; 314 | system_instruction = await transformMsg(item); 315 | } else { 316 | item.role = item.role === "assistant" ? "model" : "user"; 317 | contents.push(await transformMsg(item)); 318 | } 319 | } 320 | if (system_instruction && contents.length === 0) { 321 | contents.push({ role: "model", parts: { text: " " } }); 322 | } 323 | //console.info(JSON.stringify(contents, 2)); 324 | return { system_instruction, contents }; 325 | }; 326 | 327 | const transformRequest = async (req) => ({ 328 | ...await transformMessages(req.messages), 329 | safetySettings, 330 | generationConfig: transformConfig(req), 331 | }); 332 | 333 | const generateChatcmplId = () => { 334 | const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; 335 | const randomChar = () => characters[Math.floor(Math.random() * characters.length)]; 336 | return "chatcmpl-" + Array.from({ length: 29 }, randomChar).join(""); 337 | }; 338 | 339 | const reasonsMap = { //https://ai.google.dev/api/rest/v1/GenerateContentResponse#finishreason 340 | //"FINISH_REASON_UNSPECIFIED": // Default value. This value is unused. 341 | "STOP": "stop", 342 | "MAX_TOKENS": "length", 343 | "SAFETY": "content_filter", 344 | "RECITATION": "content_filter", 345 | //"OTHER": "OTHER", 346 | // :"function_call", 347 | }; 348 | const SEP = "\n\n|>"; 349 | const transformCandidates = (key, cand) => ({ 350 | index: cand.index || 0, // 0-index is absent in new -002 models response 351 | [key]: { 352 | role: "assistant", 353 | content: cand.content?.parts.map(p => p.text).join(SEP) }, 354 | logprobs: null, 355 | finish_reason: reasonsMap[cand.finishReason] || cand.finishReason, 356 | }); 357 | const transformCandidatesMessage = transformCandidates.bind(null, "message"); 358 | const transformCandidatesDelta = transformCandidates.bind(null, "delta"); 359 | 360 | const transformUsage = (data) => ({ 361 | completion_tokens: data.candidatesTokenCount, 362 | prompt_tokens: data.promptTokenCount, 363 | total_tokens: data.totalTokenCount 364 | }); 365 | 366 | const processCompletionsResponse = (data, model, id) => { 367 | return JSON.stringify({ 368 | id, 369 | choices: data.candidates.map(transformCandidatesMessage), 370 | created: Math.floor(Date.now()/1000), 371 | model, 372 | //system_fingerprint: "fp_69829325d0", 373 | object: "chat.completion", 374 | usage: transformUsage(data.usageMetadata), 375 | }); 376 | }; 377 | 378 | const responseLineRE = /^data: (.*)(?:\n\n|\r\r|\r\n\r\n)/; 379 | async function parseStream (chunk, controller) { 380 | chunk = await chunk; 381 | if (!chunk) { return; } 382 | this.buffer += chunk; 383 | do { 384 | const match = this.buffer.match(responseLineRE); 385 | if (!match) { break; } 386 | controller.enqueue(match[1]); 387 | this.buffer = this.buffer.substring(match[0].length); 388 | } while (true); // eslint-disable-line no-constant-condition 389 | } 390 | async function parseStreamFlush (controller) { 391 | if (this.buffer) { 392 | console.error("Invalid data:", this.buffer); 393 | controller.enqueue(this.buffer); 394 | } 395 | } 396 | 397 | function transformResponseStream (data, stop, first) { 398 | const item = transformCandidatesDelta(data.candidates[0]); 399 | if (stop) { item.delta = {}; } else { item.finish_reason = null; } 400 | if (first) { item.delta.content = ""; } else { delete item.delta.role; } 401 | const output = { 402 | id: this.id, 403 | choices: [item], 404 | created: Math.floor(Date.now()/1000), 405 | model: this.model, 406 | //system_fingerprint: "fp_69829325d0", 407 | object: "chat.completion.chunk", 408 | }; 409 | if (data.usageMetadata && this.streamIncludeUsage) { 410 | output.usage = stop ? transformUsage(data.usageMetadata) : null; 411 | } 412 | return "data: " + JSON.stringify(output) + delimiter; 413 | } 414 | const delimiter = "\n\n"; 415 | async function toOpenAiStream (chunk, controller) { 416 | const transform = transformResponseStream.bind(this); 417 | const line = await chunk; 418 | if (!line) { return; } 419 | let data; 420 | try { 421 | data = JSON.parse(line); 422 | } catch (err) { 423 | console.error(line); 424 | console.error(err); 425 | const length = this.last.length || 1; // at least 1 error msg 426 | const candidates = Array.from({ length }, (_, index) => ({ 427 | finishReason: "error", 428 | content: { parts: [{ text: err }] }, 429 | index, 430 | })); 431 | data = { candidates }; 432 | } 433 | const cand = data.candidates[0]; 434 | console.assert(data.candidates.length === 1, "Unexpected candidates count: %d", data.candidates.length); 435 | cand.index = cand.index || 0; // absent in new -002 models response 436 | if (!this.last[cand.index]) { 437 | controller.enqueue(transform(data, false, "first")); 438 | } 439 | this.last[cand.index] = data; 440 | if (cand.content) { // prevent empty data (e.g. when MAX_TOKENS) 441 | controller.enqueue(transform(data)); 442 | } 443 | } 444 | async function toOpenAiStreamFlush (controller) { 445 | const transform = transformResponseStream.bind(this); 446 | if (this.last.length > 0) { 447 | for (const data of this.last) { 448 | controller.enqueue(transform(data, "stop")); 449 | } 450 | controller.enqueue("data: [DONE]" + delimiter); 451 | } 452 | } 453 | --------------------------------------------------------------------------------