├── src ├── chat │ ├── index.ts │ ├── completions.ts │ ├── types.ts │ └── completions.test.ts ├── embeddings │ ├── index.ts │ ├── index.test.ts │ └── types.ts ├── usageTypes.ts ├── index.ts ├── request │ ├── handleStreamingResponseBody.ts │ ├── getToken.ts │ ├── index.ts │ ├── incrementalJSONParser.ts │ └── incrementalJSONParser.test.ts ├── result.ts └── completions │ ├── index.ts │ ├── types.ts │ └── index.test.ts ├── .gitignore ├── replit.nix ├── docs ├── build.sh └── extractor.json ├── README.md ├── changelog.md ├── .github └── workflows │ ├── build.yml │ └── test.yml ├── tsconfig.json ├── .eslintrc.js ├── .replit └── package.json /src/chat/index.ts: -------------------------------------------------------------------------------- 1 | export * as completions from './completions'; 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | node_modules 3 | .npmrc 4 | docs/generated 5 | .pythonlibs 6 | 7 | # repl files 8 | .breakpoints -------------------------------------------------------------------------------- /replit.nix: -------------------------------------------------------------------------------- 1 | { pkgs }: { 2 | deps = [ 3 | pkgs.htop 4 | pkgs.killall 5 | pkgs.jq.bin 6 | pkgs.nodejs-18_x 7 | ]; 8 | } -------------------------------------------------------------------------------- /docs/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | npm run clean 5 | 6 | npm run build 7 | 8 | api-extractor run -l -c docs/extractor.json 9 | 10 | api-documenter markdown -i docs/generated/api-extractor -o docs/generated/markdown 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @replit/ai-modelfarm 2 | 3 | A library for building AI applications in JavaScript and TypeScript. 4 | 5 | [Model farm general documentation](https://docs.replit.com/model-farm/) 6 | 7 | [TypeScript library API reference](https://docs.replit.com/model-farm/typescript/) 8 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | # @replit/ai changelog 2 | 3 | ## 1.0.0 4 | 5 | * Added support for v1beta2. 6 | * Introduced `provider_extra_parameters`. 7 | * Modified model requests/responses to align with widely used practices. 8 | * Consolidated `chat` and `chatStreaming` functionalities into `chat.completions.create`. 9 | * Consolidated `complete` and `completeStream` functionalities into `completions.create`. 10 | * Expanded model response interfaces. 11 | 12 | ## 0.0.3 13 | 14 | First public release 15 | -------------------------------------------------------------------------------- /src/embeddings/index.ts: -------------------------------------------------------------------------------- 1 | import * as result from '../result'; 2 | import { makeSimpleRequest, RequestError } from '../request'; 3 | import type { EmbeddingOptions, EmbeddingModelResponse } from './types'; 4 | 5 | /** 6 | * Converts text into numerical vectors 7 | * @public 8 | */ 9 | export async function create( 10 | options: EmbeddingOptions, 11 | ): Promise> { 12 | return await makeSimpleRequest('v1beta2/embeddings', { 13 | ...options, 14 | }); 15 | } 16 | -------------------------------------------------------------------------------- /src/usageTypes.ts: -------------------------------------------------------------------------------- 1 | export interface Usage { 2 | completion_tokens: number; 3 | prompt_tokens: number; 4 | total_tokens: number; 5 | } 6 | 7 | export interface TokenCountMetadata { 8 | billableTokens: number; 9 | unbilledTokens: number; 10 | billableCharacters: number; 11 | unbilledCharacters: number; 12 | } 13 | 14 | export interface GoogleMetadata { 15 | inputTokenCount?: TokenCountMetadata; 16 | outputTokenCount?: TokenCountMetadata; 17 | } 18 | 19 | export interface GoogleEmbeddingMetadata { 20 | tokenCountMetadata?: TokenCountMetadata; 21 | } 22 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Use Node.js 17 | uses: actions/setup-node@v2 18 | with: 19 | node-version: "18.x" 20 | 21 | - name: Install Dependencies 22 | run: npm ci 23 | 24 | - name: Typecheck and Build 25 | run: npm run build 26 | 27 | - name: Test the build 28 | run: npm run test:build 29 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", 4 | "strict": true, 5 | "esModuleInterop": true, 6 | "lib": ["es2022", "dom"], // adding dom here to get global fetch and friends https://github.com/microsoft/TypeScript/issues/53440 7 | "target": "ESNext", 8 | "module": "esnext", 9 | "moduleResolution": "node", 10 | "resolveJsonModule": true, 11 | "isolatedModules": true, 12 | "noUncheckedIndexedAccess": true, 13 | "skipLibCheck": true, 14 | "types": ["node"], 15 | "outDir": "dist" 16 | }, 17 | "include": [ 18 | "src/**/*.ts" 19 | ], 20 | "exclude": ["node_modules", "dist"] 21 | } -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export * as chat from './chat'; 2 | export * from './chat/types'; 3 | 4 | export * as completions from './completions'; 5 | export * from './completions/types'; 6 | 7 | export * as embeddings from './embeddings'; 8 | export * from './embeddings/types'; 9 | 10 | export * from './usageTypes'; 11 | 12 | export type { Result, OkResult, ErrResult } from './result'; 13 | export type { RequestError } from './request'; 14 | 15 | // For api-extractor, exporting the root functions 16 | export { create as completeChat } from './chat/completions'; 17 | export { create as complete } from './completions'; 18 | export { create as embed } from './embeddings'; 19 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | pull_request: 7 | branches: [main] 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Use Node.js 17 | uses: actions/setup-node@v2 18 | with: 19 | node-version: "18.x" 20 | 21 | - name: Install Dependencies 22 | run: npm ci 23 | 24 | - name: Run static checks 25 | run: npm run allstatic 26 | 27 | # Tests require to run on replit, we'll add it to PRs 28 | # via custom github webhooks 29 | # - name: Run Tests 30 | # run: npm run test 31 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | node: true, 4 | es2021: true, 5 | }, 6 | extends: [ 7 | 'eslint:recommended', 8 | 'plugin:@typescript-eslint/strict-type-checked', 9 | 'plugin:@typescript-eslint/stylistic-type-checked', 10 | 'plugin:prettier/recommended', 11 | ], 12 | parser: '@typescript-eslint/parser', 13 | parserOptions: { 14 | ecmaVersion: 'latest', 15 | sourceType: 'module', 16 | project: ['./tsconfig.json'], 17 | }, 18 | plugins: ['@typescript-eslint'], 19 | rules: { 20 | 'linebreak-style': ['error', 'unix'], 21 | '@typescript-eslint/array-type': ['error', { default: 'generic' }], 22 | }, 23 | ignorePatterns: ['/*', '!/src'], 24 | }; 25 | -------------------------------------------------------------------------------- /src/embeddings/index.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-assignment*/ 2 | 3 | import { expect, test } from 'vitest'; 4 | import * as replitai from '../index'; 5 | 6 | test('embed', async () => { 7 | const { error, value: embedding } = await replitai.embeddings.create({ 8 | model: 'textembedding-gecko', 9 | input: ['how to quit in vim', 'how to quit in emacs'], 10 | }); 11 | 12 | expect(error).toBeFalsy(); 13 | expect(embedding).not.toBeUndefined(); 14 | 15 | expect(embedding?.data.length).toBe(2); 16 | for (const idx of [0, 1]) { 17 | expect(embedding?.data[idx]).toMatchObject( 18 | expect.objectContaining({ 19 | index: idx, 20 | embedding: expect.arrayContaining([expect.any(Number)]), 21 | }), 22 | ); 23 | } 24 | }); 25 | -------------------------------------------------------------------------------- /docs/extractor.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json", 3 | "mainEntryPointFilePath": "/dist/index.d.ts", 4 | "bundledPackages": [], 5 | "compiler": {}, 6 | "apiReport": { 7 | "enabled": true, 8 | "reportFolder": "/docs/generated/api-extractor", 9 | "reportFileName": "api-report.md", 10 | "reportTempFolder": "/tmp//api-extractor", 11 | }, 12 | "docModel": { 13 | "enabled": true, 14 | "apiJsonFilePath": "/docs/generated/api-extractor/api-model.api.json" 15 | }, 16 | "dtsRollup": { 17 | "enabled": false 18 | }, 19 | "tsdocMetadata": {}, 20 | "messages": { 21 | "compilerMessageReporting": { 22 | "default": { 23 | "logLevel": "warning" 24 | } 25 | }, 26 | "extractorMessageReporting": { 27 | "default": { 28 | "logLevel": "warning" 29 | } 30 | }, 31 | "tsdocMessageReporting": { 32 | "default": { 33 | "logLevel": "warning" 34 | } 35 | } 36 | } 37 | } -------------------------------------------------------------------------------- /.replit: -------------------------------------------------------------------------------- 1 | entrypoint = "index.js" 2 | run = "npm run test:watch" 3 | 4 | modules = ["python-3.10:v18-20230807-322e88b"] 5 | 6 | hidden = [".config", "package-lock.json"] 7 | 8 | [nix] 9 | channel = "stable-22_11" 10 | 11 | [env] 12 | XDG_CONFIG_HOME = "$REPL_HOME/.config" 13 | PATH = "$REPL_HOME/.config/npm/node_global/bin:$REPL_HOME/node_modules/.bin" 14 | npm_config_prefix = "$REPL_HOME/.config/npm/node_global" 15 | 16 | [packager] 17 | language = "nodejs" 18 | 19 | [packager.features] 20 | packageSearch = true 21 | guessImports = false 22 | enabledForHosting = false 23 | 24 | [languages] 25 | [languages.javascript] 26 | pattern = "**/{*.js,*.jsx,*.ts,*.tsx,*.json}" 27 | [languages.javascript.languageServer] 28 | start = "./node_modules/.bin/typescript-language-server --stdio" 29 | [[languages.javascript.languageServer.initializationOptions.plugins]] 30 | name = "typescript-eslint-language-service" 31 | location = "./node_modules/typescript-eslint-language-service" 32 | 33 | [deployment] 34 | build = ["sh", "-c", "npm run build:docs"] 35 | deploymentTarget = "static" 36 | publicDir = "docs/generated/website" 37 | -------------------------------------------------------------------------------- /src/request/handleStreamingResponseBody.ts: -------------------------------------------------------------------------------- 1 | import { pipe } from 'it-pipe'; 2 | import streamToIterator from 'browser-readablestream-to-it'; 3 | import IncrementalJSONParser from './incrementalJSONParser'; 4 | 5 | export default function handleStreamingResponseBody( 6 | responseBody: ReadableStream, 7 | ): AsyncGenerator { 8 | return pipe( 9 | streamToIterator(responseBody), 10 | async function* (source) { 11 | const decoder = new TextDecoder('utf-8'); 12 | 13 | for await (const v of source) { 14 | yield decoder.decode(v, { stream: true }); 15 | } 16 | 17 | return; 18 | }, 19 | async function* (source): AsyncGenerator { 20 | const parser = new IncrementalJSONParser(); 21 | for await (const v of source) { 22 | const parserSource = parser.write(v); 23 | 24 | for (const json of parserSource) { 25 | // Ideally we do some assertions on T here 26 | yield json as R; 27 | } 28 | } 29 | 30 | if (parser.hasPending()) { 31 | throw new Error('stream ended with unfinished data'); 32 | } 33 | }, 34 | ); 35 | } 36 | -------------------------------------------------------------------------------- /src/embeddings/types.ts: -------------------------------------------------------------------------------- 1 | import type { Usage, GoogleEmbeddingMetadata } from '../usageTypes'; 2 | 3 | export interface Embedding { 4 | object: string; 5 | embedding: Array; 6 | index: number; 7 | metadata?: Record; 8 | } 9 | 10 | export interface EmbeddingModelResponse { 11 | object: string; 12 | data: Array; 13 | model: string; 14 | usage?: Usage; 15 | metadata?: GoogleEmbeddingMetadata; 16 | } 17 | 18 | /** 19 | * Options for embedding request 20 | */ 21 | export interface EmbeddingOptions { 22 | /** 23 | * The model to embed with 24 | */ 25 | model: string; 26 | /** 27 | * The strings to embed, the returned embedding will correspond to the order 28 | * of the passed string 29 | */ 30 | input: string | Array | Array | Array>; 31 | 32 | /** 33 | * Index signature allowing any other options 34 | */ 35 | [key: string]: unknown; 36 | 37 | /** 38 | * Allows extra provider specific parameters. Consult with the documentation for which 39 | * parameters are available for each model. 40 | */ 41 | provider_extra_parameters?: Record; 42 | } 43 | -------------------------------------------------------------------------------- /src/result.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * A Result type that can be used to represent a successful value or an error. 3 | * It forces the consumer to check whether the returned type is an error or not, 4 | * `result.ok` acts as a discriminant between success and failure 5 | * @public 6 | */ 7 | export type Result = 8 | | OkResult 9 | | ErrResult; 10 | 11 | /** 12 | * Represents a successful result 13 | * @public 14 | */ 15 | export interface OkResult { 16 | ok: true; 17 | value: T; 18 | error?: undefined; 19 | } 20 | 21 | /** 22 | * Represents a failure result 23 | * @public 24 | */ 25 | export interface ErrResult { 26 | ok: false; 27 | error: E; 28 | value?: undefined; 29 | errorExtras?: ErrorExtras; 30 | } 31 | 32 | /** 33 | * A helper function to create an error Result type 34 | */ 35 | export function Err( 36 | error: E, 37 | errorExtras?: ErrorExtras, 38 | ): ErrResult { 39 | return { ok: false, error, errorExtras }; 40 | } 41 | 42 | /** 43 | * A helper function to create a successful Result type 44 | **/ 45 | export function Ok(value: T): OkResult { 46 | return { ok: true, value }; 47 | } 48 | -------------------------------------------------------------------------------- /src/completions/index.ts: -------------------------------------------------------------------------------- 1 | import * as result from '../result'; 2 | import { 3 | makeStreamingRequest, 4 | makeSimpleRequest, 5 | RequestError, 6 | } from '../request'; 7 | import type { 8 | CompletionOptionsStream, 9 | CompletionResponse, 10 | CompletionOptionsNonStream, 11 | CompletionOptions, 12 | } from './types'; 13 | 14 | /** 15 | * Gets the completion for a piece of text. 16 | * @public 17 | */ 18 | export async function create( 19 | options: CompletionOptionsNonStream, 20 | ): Promise>; 21 | /** 22 | * Gets a stream of completions for a piece of text. 23 | * @public 24 | */ 25 | export async function create( 26 | options: CompletionOptionsStream, 27 | ): Promise, RequestError>>; 28 | export async function create( 29 | options: CompletionOptions, 30 | ): Promise< 31 | result.Result< 32 | CompletionResponse | AsyncGenerator, 33 | RequestError 34 | > 35 | > { 36 | if (options.stream) { 37 | return await makeStreamingRequest( 38 | 'v1beta2/completions', 39 | { ...options }, 40 | ); 41 | } else { 42 | return await makeSimpleRequest('v1beta2/completions', { 43 | ...options, 44 | }); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/chat/completions.ts: -------------------------------------------------------------------------------- 1 | import * as result from '../result'; 2 | import { 3 | makeStreamingRequest, 4 | makeSimpleRequest, 5 | RequestError, 6 | } from '../request'; 7 | import type { 8 | ChatOptionParamsNonStream, 9 | ChatCompletionResponse, 10 | ChatOptionParamsStream, 11 | ChatCompletionStreamChunkResponse, 12 | ChatOptionParams, 13 | } from './types'; 14 | 15 | /** 16 | * Gets a single chat message completion for a conversation. 17 | * @public 18 | */ 19 | export async function create( 20 | options: ChatOptionParamsNonStream, 21 | ): Promise>; 22 | /** 23 | * Gets a single chat message completion for a conversation. 24 | * The result contains an iterator of messages, please note that this would be 25 | * a *single message* that has the contents chunked up. 26 | * @public 27 | */ 28 | export async function create( 29 | options: ChatOptionParamsStream, 30 | ): Promise< 31 | result.Result, RequestError> 32 | >; 33 | export async function create( 34 | options: ChatOptionParams, 35 | ): Promise< 36 | result.Result< 37 | ChatCompletionResponse | AsyncGenerator, 38 | RequestError 39 | > 40 | > { 41 | if (options.stream) { 42 | return await makeStreamingRequest( 43 | 'v1beta2/chat/completions', 44 | { ...options }, 45 | ); 46 | } else { 47 | return await makeSimpleRequest( 48 | 'v1beta2/chat/completions', 49 | { ...options }, 50 | ); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/completions/types.ts: -------------------------------------------------------------------------------- 1 | import type { GoogleMetadata, Usage } from '../usageTypes'; 2 | 3 | export interface CompletionChoice { 4 | index: number; 5 | text: string; 6 | finish_reason: string; 7 | logprobs?: Record; 8 | metadata?: Record; 9 | } 10 | 11 | export interface CompletionResponse { 12 | id: string; 13 | choices: Array; 14 | model: string; 15 | created?: number; 16 | object?: string; 17 | usage?: Usage; 18 | metadata?: GoogleMetadata; 19 | } 20 | 21 | /** 22 | * Options for completion request 23 | * @public 24 | */ 25 | export interface CompletionOptionsBase { 26 | /** 27 | * Specifies the model to use 28 | */ 29 | model: string; 30 | /** 31 | * The string/text to complete 32 | */ 33 | prompt: string | Array | Array | Array> | null; 34 | /** 35 | * Sampling temperature. The higher the value, the more 36 | * likely the model will produce a completion that is more creative and 37 | * imaginative. 38 | */ 39 | temperature?: number; 40 | /** 41 | * The maximum number of tokens generated in the completion. 42 | * The absolute maximum value is limited by model's context size. 43 | */ 44 | max_tokens?: number; 45 | 46 | /** 47 | * Wheter to stream the completions. 48 | */ 49 | stream?: boolean; 50 | 51 | /** 52 | * Index signature allowing any other options 53 | */ 54 | [key: string]: unknown; 55 | 56 | /** 57 | * Allows extra provider specific parameters. Consult with the documentation for which 58 | * parameters are available for each model. 59 | */ 60 | provider_extra_parameters?: Record; 61 | } 62 | 63 | export interface CompletionOptionsStream extends CompletionOptionsBase { 64 | stream: true; 65 | } 66 | 67 | export interface CompletionOptionsNonStream extends CompletionOptionsBase { 68 | stream?: false; 69 | } 70 | 71 | export type CompletionOptions = 72 | | CompletionOptionsStream 73 | | CompletionOptionsNonStream; 74 | -------------------------------------------------------------------------------- /src/request/getToken.ts: -------------------------------------------------------------------------------- 1 | import { exec } from 'child_process'; 2 | 3 | const isDeployment = Boolean(process.env.REPLIT_DEPLOYMENT); 4 | 5 | async function genReplIdentityToken(): Promise { 6 | return new Promise((resolve, reject) => { 7 | exec( 8 | '$REPLIT_CLI identity create -audience="modelfarm@replit.com"', 9 | (error, stdout, stderr) => { 10 | if (error) { 11 | reject(`Getting identity token: ${error.name} ${error.message}`); 12 | return; 13 | } 14 | 15 | if (stderr) { 16 | reject(`Saw stderr getting identity token: ${stderr}`); 17 | return; 18 | } 19 | 20 | resolve(stdout.trim()); 21 | }, 22 | ); 23 | }); 24 | } 25 | 26 | async function getDeploymentToken(): Promise { 27 | const res = await fetch('http://127.0.0.1:1105/getIdentityToken', { 28 | body: JSON.stringify({ audience: 'modelfarm@replit.com' }), 29 | headers: { 30 | 'Content-Type': 'application/json', 31 | }, 32 | method: 'POST', 33 | }); 34 | 35 | const json = (await res.json()) as unknown; 36 | 37 | if (!json) { 38 | throw new Error('Expected json to have identity token'); 39 | } 40 | 41 | if (typeof json !== 'object') { 42 | throw new Error('Expected json to be an object'); 43 | } 44 | 45 | if (!('identityToken' in json)) { 46 | throw new Error('Expected json to have identity token'); 47 | } 48 | 49 | if (typeof json.identityToken !== 'string') { 50 | throw new Error('Expected identity token to be a string'); 51 | } 52 | 53 | return json.identityToken; 54 | } 55 | 56 | let cachedToken: string | null = null; 57 | 58 | function resetTokenSoon() { 59 | setTimeout(() => { 60 | cachedToken = null; 61 | }, 1000 * 30); 62 | } 63 | 64 | export default async function getToken() { 65 | if (!cachedToken) { 66 | const fn = isDeployment ? getDeploymentToken : genReplIdentityToken; 67 | cachedToken = await fn(); 68 | resetTokenSoon(); 69 | } 70 | 71 | return cachedToken; 72 | } 73 | -------------------------------------------------------------------------------- /src/request/index.ts: -------------------------------------------------------------------------------- 1 | import * as result from '../result'; 2 | import getToken from './getToken'; 3 | import handleStreamingResponseBody from './handleStreamingResponseBody'; 4 | 5 | /** 6 | * An object that represents an error with a request 7 | * @public 8 | */ 9 | export interface RequestError { 10 | message: string; 11 | statusCode: number; 12 | } 13 | 14 | export const baseUrl = 15 | process.env.MODEL_FARM_URL ?? 'https://production-modelfarm.replit.com/'; 16 | 17 | export async function doFetch( 18 | urlPath: string, 19 | body: Record, 20 | ): Promise< 21 | result.Result< 22 | Response & { body: NonNullable }, 23 | RequestError 24 | > 25 | > { 26 | const url = new URL(urlPath, baseUrl); 27 | 28 | const response = await fetch(url, { 29 | method: 'POST', 30 | headers: { 31 | 'Content-Type': 'application/json', 32 | Authorization: `Bearer ${await getToken()}`, 33 | }, 34 | body: JSON.stringify(body), 35 | }); 36 | 37 | if (response.status !== 200) { 38 | return result.Err({ 39 | message: await response.text(), 40 | statusCode: response.status, 41 | }); 42 | } 43 | 44 | if (!response.body) { 45 | return result.Err({ 46 | message: 'No response body', 47 | statusCode: response.status, 48 | }); 49 | } 50 | 51 | return result.Ok( 52 | response as Response & { body: NonNullable }, 53 | ); 54 | } 55 | 56 | export async function makeStreamingRequest( 57 | urlPath: string, 58 | body: Record, 59 | ): Promise, RequestError>> { 60 | const res = await doFetch(urlPath, body); 61 | 62 | if (!res.ok) { 63 | return res; 64 | } 65 | 66 | const iterator = handleStreamingResponseBody(res.value.body); 67 | 68 | return result.Ok(iterator); 69 | } 70 | 71 | export async function makeSimpleRequest( 72 | urlPath: string, 73 | body: Record, 74 | ): Promise> { 75 | const res = await doFetch(urlPath, body); 76 | 77 | if (!res.ok) { 78 | return res; 79 | } 80 | 81 | const json = (await res.value.json()) as R; 82 | 83 | return result.Ok(json); 84 | } 85 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@replit/ai-modelfarm", 3 | "version": "1.0.0", 4 | "description": "A library for building AI applications in JavaScript and TypeScript.", 5 | "homepage": "https://docs.replit.com/ai/model-farm/", 6 | "repository": "https://github.com/replit/replit-ai-modelfarm-typescript", 7 | "engines": { 8 | "node": ">=18.0.0" 9 | }, 10 | "scripts": { 11 | "test": "vitest run src/*", 12 | "test:watch": "vitest src/*", 13 | "test:build": "vitest run buildtests/*", 14 | "lint": "eslint src/*", 15 | "lint:fix": "eslint src/* --fix", 16 | "format": "prettier src/* --check", 17 | "format:fix": "prettier src/* --write", 18 | "fix": "npm run format:fix && npm run lint:fix", 19 | "typecheck": "tsc --noEmit", 20 | "allstatic": "npm run typecheck && npm run lint && npm run format", 21 | "clean": "rm -rf dist docs/generated", 22 | "build": "tsup src/index.ts --sourcemap --dts --platform node --format esm,cjs", 23 | "build:docs": "./docs/build.sh", 24 | "prepublishOnly": "npm run allstatic && npm run test && npm run build && npm run test:build" 25 | }, 26 | "exports": { 27 | "./package.json": "./package.json", 28 | ".": { 29 | "types": "./dist/index.d.ts", 30 | "import": { 31 | "default": "./dist/index.mjs" 32 | }, 33 | "require": "./dist/index.js", 34 | "default": "./dist/index.js" 35 | } 36 | }, 37 | "main": "./dist/index.js", 38 | "types": "./dist/index.d.ts", 39 | "keywords": [ 40 | "replit", 41 | "ai" 42 | ], 43 | "files": [ 44 | "dist/*" 45 | ], 46 | "author": "", 47 | "license": "Apache-2.0", 48 | "devDependencies": { 49 | "@jest/globals": "^29.6.4", 50 | "@microsoft/api-documenter": "^7.22.33", 51 | "@microsoft/api-extractor": "^7.36.4", 52 | "@types/node": "^18.0.6", 53 | "@typescript-eslint/eslint-plugin": "^6.6.0", 54 | "@typescript-eslint/parser": "^6.6.0", 55 | "browser-readablestream-to-it": "^2.0.4", 56 | "eslint": "^8.48.0", 57 | "eslint-config-prettier": "^9.0.0", 58 | "eslint-plugin-prettier": "^5.0.0", 59 | "it-pipe": "^3.0.1", 60 | "jest": "^29.6.4", 61 | "prettier": "^3.0.3", 62 | "ts-jest": "^29.1.1", 63 | "tsup": "^7.2.0", 64 | "typescript": "^5.2.2", 65 | "typescript-eslint-language-service": "^5.0.5", 66 | "typescript-language-server": "^3.3.2", 67 | "vitest": "^0.34.3" 68 | }, 69 | "prettier": { 70 | "trailingComma": "all", 71 | "singleQuote": true 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/completions/index.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-assignment*/ 2 | 3 | import { expect, test } from 'vitest'; 4 | import * as replitai from '../index'; 5 | 6 | async function fromAsync( 7 | source: AsyncIterable | undefined, 8 | ): Promise> { 9 | const items = Array(); 10 | if (source) { 11 | for await (const item of source) { 12 | items.push(item); 13 | } 14 | } 15 | return items; 16 | } 17 | 18 | test('non streaming completion', async () => { 19 | const { error, value: result } = await replitai.completions.create({ 20 | model: 'text-bison', 21 | prompt: 22 | "Here's an essay about why the chicken crossed the road\n # The Chicken and The Road\n", 23 | temperature: 0.5, 24 | max_tokens: 128, 25 | }); 26 | 27 | expect(error).toBeFalsy(); 28 | expect(result).not.toBeUndefined(); 29 | 30 | expect(result?.choices[0]?.text).toEqual(expect.any(String)); 31 | }); 32 | 33 | test('non streaming completion with extra parameters', async () => { 34 | const { error, value: result } = await replitai.completions.create({ 35 | model: 'text-bison', 36 | prompt: 'Complete this sequence up to 10: 1, 2, 3, 4, 5', 37 | temperature: 0.5, 38 | max_tokens: 128, 39 | stop: ['7', '7,'], 40 | }); 41 | 42 | expect(error).toBeFalsy(); 43 | expect(result).not.toBeUndefined(); 44 | 45 | expect(result?.choices[0]?.text).toEqual(expect.any(String)); 46 | 47 | expect(result?.choices[0]?.text.includes('6')).toBeTruthy(); 48 | expect(result?.choices[0]?.text.includes('7')).toBeFalsy(); 49 | }); 50 | 51 | test('streaming completion', async () => { 52 | const { error, value: results } = await replitai.completions.create({ 53 | model: 'text-bison', 54 | prompt: 55 | "Here's an essay about why the chicken crossed the road\n # The Chicken and The Road\n", 56 | temperature: 0.5, 57 | max_tokens: 128, 58 | stream: true, 59 | }); 60 | 61 | expect(error).toBeFalsy(); 62 | expect(results).not.toBeUndefined(); 63 | 64 | const responses = await fromAsync(results); 65 | 66 | for await (const completion of responses) { 67 | expect(completion.choices[0]?.text).toEqual(expect.any(String)); 68 | } 69 | }); 70 | 71 | test('completion with multiple choices', async () => { 72 | const { error, value: result } = await replitai.completions.create({ 73 | model: 'text-bison', 74 | prompt: 75 | "Here's an essay about why the chicken crossed the road\n # The Chicken and The Road\n", 76 | temperature: 1, 77 | max_tokens: 128, 78 | n: 4, 79 | }); 80 | 81 | expect(error).toBeFalsy(); 82 | expect(result).not.toBeUndefined(); 83 | 84 | expect(result).toMatchObject({ 85 | choices: expect.arrayContaining([ 86 | expect.objectContaining({ 87 | text: expect.any(String), 88 | }), 89 | ]), 90 | }); 91 | expect(result?.choices.length).toBeGreaterThan(1); 92 | }); 93 | -------------------------------------------------------------------------------- /src/chat/types.ts: -------------------------------------------------------------------------------- 1 | import type { GoogleMetadata, Usage } from '../usageTypes'; 2 | 3 | export interface FunctionCall { 4 | name: string; 5 | arguments: string; 6 | } 7 | 8 | export interface ToolCall { 9 | id: string; 10 | type: string; 11 | function: FunctionCall; 12 | } 13 | 14 | export interface ChoiceMessage { 15 | content?: string; 16 | role?: string; 17 | tool_calls?: Array; 18 | } 19 | 20 | export interface BaseChoice { 21 | index: number; 22 | finish_reason?: string; 23 | metadata?: Record; 24 | } 25 | 26 | export interface Choice extends BaseChoice { 27 | message: ChoiceMessage; 28 | } 29 | 30 | export interface ChoiceStream extends BaseChoice { 31 | delta: ChoiceMessage; 32 | } 33 | 34 | export interface BaseChatCompletionResponse { 35 | id: string; 36 | choices: Array; 37 | model: string; 38 | created?: number; 39 | object?: string; 40 | usage?: Usage; 41 | metadata?: GoogleMetadata; 42 | } 43 | 44 | export interface ChatCompletionResponse extends BaseChatCompletionResponse { 45 | choices: Array; 46 | object?: string; 47 | } 48 | 49 | export interface ChatCompletionStreamChunkResponse 50 | extends BaseChatCompletionResponse { 51 | choices: Array; 52 | object?: string; 53 | } 54 | 55 | export interface ChatCompletionMessageRequestParam { 56 | /** 57 | * The author of the message. 58 | * Typically the completion infers the author from examples and previous 59 | * messages provided in the options. 60 | */ 61 | role?: string; 62 | /** 63 | * The content of the message 64 | */ 65 | content?: string; 66 | 67 | tool_calls?: Array; 68 | tool_call_id?: string; 69 | } 70 | 71 | /** 72 | * Options for chat request 73 | * @public 74 | */ 75 | export interface ChatOptionParamsBase { 76 | /** 77 | * Specifies the model to use 78 | */ 79 | model: string; 80 | /** 81 | * Previous messages in the conversation 82 | */ 83 | messages: Array; 84 | /** 85 | * Sampling temperature between 0 and 1. The higher the value, the more 86 | * likely the model will produce a completion that is more creative and 87 | * imaginative. 88 | */ 89 | temperature?: number; 90 | /** 91 | * The maximum number of tokens generated in the chat completion. 92 | * The absolute maximum value is limited by model's context size. 93 | */ 94 | max_tokens?: number; 95 | 96 | /** 97 | * Wheter to stream the completions. 98 | */ 99 | stream?: boolean; 100 | 101 | /** 102 | * Index signature allowing any other options 103 | */ 104 | [key: string]: unknown; 105 | 106 | /** 107 | * Allows extra provider specific parameters. Consult with the documentation for which 108 | * parameters are available for each model. 109 | */ 110 | provider_extra_parameters?: Record; 111 | } 112 | 113 | export interface ChatOptionParamsStream extends ChatOptionParamsBase { 114 | /** 115 | * Wheter to stream the completions. 116 | */ 117 | stream: true; 118 | } 119 | 120 | export interface ChatOptionParamsNonStream extends ChatOptionParamsBase { 121 | /** 122 | * Wheter to stream the completions. 123 | */ 124 | stream?: false; 125 | } 126 | 127 | export type ChatOptionParams = 128 | | ChatOptionParamsStream 129 | | ChatOptionParamsNonStream; 130 | -------------------------------------------------------------------------------- /src/request/incrementalJSONParser.ts: -------------------------------------------------------------------------------- 1 | enum State { 2 | EMPTY = 0, 3 | IN_DOUBLE_QUOTE_STRING = 2, 4 | NORMAL = 1, 5 | } 6 | 7 | /** 8 | * This parses a stream of JSON that may be incomplete 9 | * after every chunk 10 | */ 11 | export default class IncrementalJSONParser { 12 | private buffer: string; 13 | private bufferIndex: number; 14 | private state: State; 15 | private level: number; 16 | private escaped: boolean; 17 | private locked: boolean; 18 | 19 | constructor() { 20 | this.buffer = ''; 21 | this.bufferIndex = 0; 22 | // State at the last read character. 23 | this.state = State.EMPTY; 24 | this.level = 0; 25 | this.escaped = false; 26 | this.locked = false; 27 | } 28 | 29 | public write(chunk: string) { 30 | if (this.locked) { 31 | throw new Error('parser locked, make sure to drain the previous write'); 32 | } 33 | 34 | this.locked = true; 35 | 36 | return this.writeGenerator(chunk); 37 | } 38 | 39 | /** 40 | * Is the written string fully consumed or we have buffered 41 | * data that needs to be processed 42 | */ 43 | public hasPending() { 44 | return this.buffer.length > 0; 45 | } 46 | 47 | /** 48 | * Did we call write() and the iterator is not exhausted yet 49 | */ 50 | public isLocked() { 51 | return this.locked; 52 | } 53 | 54 | private *writeGenerator(chunk: string): Generator { 55 | // Push the data into the system, and then process any pending content 56 | const workData = this.buffer + chunk; 57 | let lastLevel0 = 0; 58 | while (this.bufferIndex < workData.length) { 59 | if (this.state === State.IN_DOUBLE_QUOTE_STRING) { 60 | // Look for the next unescaped quote 61 | for (; this.bufferIndex < workData.length; this.bufferIndex++) { 62 | const c = workData[this.bufferIndex]; 63 | if (this.escaped) { 64 | this.escaped = false; 65 | } else if (c === '\\') { 66 | this.escaped = true; 67 | } else if (c === '"') { 68 | this.state = State.NORMAL; 69 | this.bufferIndex++; 70 | break; 71 | } 72 | } 73 | } else { 74 | // Process content regularly until we find a string start 75 | for (; this.bufferIndex < workData.length; this.bufferIndex++) { 76 | const c = workData[this.bufferIndex]; 77 | if (c === '{') { 78 | this.level++; 79 | } else if (c === '}') { 80 | this.level--; 81 | if (this.level === 0) { 82 | // Parse the block until now 83 | const parsed = JSON.parse( 84 | workData.substring(lastLevel0, this.bufferIndex + 1), 85 | ) as T; 86 | yield parsed; 87 | 88 | // Reset buffer to the section from now 89 | lastLevel0 = this.bufferIndex + 1; 90 | } 91 | } else if (c === '"') { 92 | this.state = State.IN_DOUBLE_QUOTE_STRING; 93 | this.bufferIndex++; 94 | break; 95 | } 96 | } 97 | } 98 | } 99 | 100 | if (lastLevel0 > 0) { 101 | this.buffer = workData.substring(lastLevel0); 102 | this.bufferIndex -= lastLevel0; 103 | } else { 104 | this.buffer = workData; 105 | } 106 | 107 | this.locked = false; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/chat/completions.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-unsafe-assignment*/ 2 | 3 | import { expect, test } from 'vitest'; 4 | import * as replitai from '../index'; 5 | 6 | async function fromAsync( 7 | source: AsyncIterable | undefined, 8 | ): Promise> { 9 | const items = Array(); 10 | if (source) { 11 | for await (const item of source) { 12 | items.push(item); 13 | } 14 | } 15 | return items; 16 | } 17 | 18 | test('non streaming chat', async () => { 19 | const { error, value: result } = await replitai.chat.completions.create({ 20 | model: 'chat-bison', 21 | messages: [ 22 | { 23 | role: 'USER', 24 | content: 'What is the meaning of life?', 25 | }, 26 | ], 27 | temperature: 0.5, 28 | max_tokens: 128, 29 | }); 30 | 31 | expect(error).toBeFalsy(); 32 | expect(result).not.toBeUndefined(); 33 | 34 | expect(result?.choices.length).toBe(1); 35 | 36 | const choice = result?.choices[0]; 37 | 38 | expect(choice?.message.content?.length).toBeGreaterThan(10); 39 | }); 40 | 41 | test('non streaming chat with extra parameters', async () => { 42 | const { error, value: result } = await replitai.chat.completions.create({ 43 | model: 'chat-bison', 44 | messages: [ 45 | { 46 | role: 'USER', 47 | content: 'What is the meaning of life?', 48 | }, 49 | ], 50 | temperature: 0.0, 51 | max_tokens: 1024, 52 | stop: ['\n'], 53 | top_p: 0.1, 54 | provider_extra_parameters: { 55 | top_k: 20, 56 | }, 57 | }); 58 | 59 | expect(error).toBeFalsy(); 60 | expect(result).not.toBeUndefined(); 61 | 62 | expect(result?.choices.length).toBe(1); 63 | 64 | const choice = result?.choices[0]; 65 | 66 | expect(choice?.message.content?.length).toBeGreaterThan(10); 67 | }); 68 | 69 | test('streaming chat', async () => { 70 | const { error, value: results } = await replitai.chat.completions.create({ 71 | model: 'chat-bison', 72 | messages: [ 73 | { 74 | role: 'USER', 75 | content: 'What is the meaning of life?', 76 | }, 77 | ], 78 | temperature: 0.5, 79 | max_tokens: 128, 80 | stream: true, 81 | }); 82 | 83 | expect(error).toBeFalsy(); 84 | expect(results).not.toBeUndefined(); 85 | 86 | const responses = await fromAsync(results); 87 | 88 | for await (const result of responses) { 89 | expect(result.choices.length).toBe(1); 90 | const choice = result.choices[0]; 91 | expect(choice?.delta.content?.length).toBeGreaterThanOrEqual(0); 92 | } 93 | }); 94 | 95 | test('chat with multiple choices', async () => { 96 | const { error, value: result } = await replitai.chat.completions.create({ 97 | model: 'chat-bison', 98 | messages: [ 99 | { 100 | role: 'USER', 101 | content: 'What is the meaning of life?', 102 | }, 103 | ], 104 | temperature: 1, 105 | max_tokens: 128, 106 | n: 4, 107 | }); 108 | 109 | expect(error).toBeFalsy(); 110 | expect(result).not.toBeUndefined(); 111 | 112 | expect(result).toMatchObject( 113 | expect.objectContaining({ 114 | choices: expect.arrayContaining([ 115 | expect.objectContaining({ 116 | message: expect.objectContaining({ 117 | content: expect.any(String), 118 | role: expect.any(String), 119 | }), 120 | }), 121 | ]), 122 | }), 123 | ); 124 | expect(result?.choices.length).toBeGreaterThan(1); 125 | }); 126 | -------------------------------------------------------------------------------- /src/request/incrementalJSONParser.test.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from 'vitest'; 2 | import IncrementalJSONParser from './incrementalJSONParser'; 3 | 4 | test('reports pending when incomplete', () => { 5 | const parser = new IncrementalJSONParser(); 6 | const next = parser.write('{').next(); 7 | expect(next.done).toBeTruthy(); 8 | expect(next.value).toBeFalsy(); 9 | expect(parser.hasPending()).toBeTruthy(); 10 | expect(parser.isLocked()).toBeFalsy(); 11 | }); 12 | 13 | test('reports not pending after construction', () => { 14 | const parser = new IncrementalJSONParser(); 15 | 16 | expect(parser.isLocked()).toBeFalsy(); 17 | expect(parser.hasPending()).toBeFalsy(); 18 | }); 19 | 20 | test('loads simple json', () => { 21 | const parser = new IncrementalJSONParser(); 22 | 23 | const iter = parser.write('{}'); 24 | expect(parser.hasPending()).toBeFalsy(); 25 | expect(parser.isLocked()).toBeTruthy(); 26 | 27 | const next = iter.next(); 28 | expect(next.value).toMatchObject({}); 29 | expect(next.done).toBeFalsy(); 30 | expect(iter.next().done).toBeTruthy(); 31 | 32 | expect(parser.isLocked()).toBeFalsy(); 33 | expect(parser.hasPending()).toBeFalsy(); 34 | }); 35 | 36 | test('loads chunked json (single write)', () => { 37 | const parser = new IncrementalJSONParser(); 38 | 39 | const iter = parser.write('{}{}'); 40 | 41 | const next = iter.next(); 42 | expect(next.value).toMatchObject({}); 43 | expect(next.done).toBeFalsy(); 44 | 45 | const nextnext = iter.next(); 46 | expect(nextnext.value).toMatchObject({}); 47 | expect(nextnext.done).toBeFalsy(); 48 | expect(iter.next().done).toBeTruthy(); 49 | 50 | expect(parser.isLocked()).toBeFalsy(); 51 | expect(parser.hasPending()).toBeFalsy(); 52 | }); 53 | 54 | test('loads chunked json (multiple wrtes)', () => { 55 | const parser = new IncrementalJSONParser(); 56 | 57 | const iter1 = parser.write('{}'); 58 | expect(iter1.next().value).toMatchObject({}); 59 | expect(iter1.next().done).toBeTruthy(); 60 | 61 | const iter2 = parser.write('{}'); 62 | expect(iter2.next().value).toMatchObject({}); 63 | expect(iter2.next().done).toBeTruthy(); 64 | 65 | expect(parser.isLocked()).toBeFalsy(); 66 | expect(parser.hasPending()).toBeFalsy(); 67 | }); 68 | 69 | test('loads chunked json with extra whitespace', () => { 70 | const parser = new IncrementalJSONParser(); 71 | 72 | const iter1 = parser.write('{}'); 73 | expect(iter1.next().value).toMatchObject({}); 74 | expect(iter1.next().done).toBeTruthy(); 75 | 76 | const iter2 = parser.write(' \r\n\t'); 77 | expect(iter2.next().done).toBeTruthy(); 78 | 79 | const iter3 = parser.write('{}'); 80 | expect(iter3.next().value).toMatchObject({}); 81 | expect(iter3.next().done).toBeTruthy(); 82 | 83 | expect(parser.isLocked()).toBeFalsy(); 84 | expect(parser.hasPending()).toBeFalsy(); 85 | }); 86 | 87 | test('handles odd boundary (non-string)', () => { 88 | const parser = new IncrementalJSONParser(); 89 | 90 | const iter1 = parser.write('{"field":"value"'); 91 | expect(iter1.next().done).toBeTruthy(); 92 | 93 | const iter2 = parser.write('}'); 94 | expect(iter2.next().value).toMatchObject({ field: 'value' }); 95 | expect(iter2.next().done).toBeTruthy(); 96 | 97 | expect(parser.isLocked()).toBeFalsy(); 98 | expect(parser.hasPending()).toBeFalsy(); 99 | }); 100 | 101 | test('handles odd boundary (double-quoted-string)', () => { 102 | const parser = new IncrementalJSONParser(); 103 | 104 | const iter1 = parser.write('{"field"'); 105 | expect(iter1.next().done).toBeTruthy(); 106 | 107 | const iter2 = parser.write(':"value"}'); 108 | expect(iter2.next().value).toMatchObject({ field: 'value' }); 109 | expect(iter2.next().done).toBeTruthy(); 110 | 111 | expect(parser.isLocked()).toBeFalsy(); 112 | expect(parser.hasPending()).toBeFalsy(); 113 | }); 114 | 115 | test('handles odd boundary (double-quoted-string, escaped)', () => { 116 | const parser = new IncrementalJSONParser(); 117 | 118 | const iter1 = parser.write('{"field": "value\\"}'); 119 | expect(iter1.next().done).toBeTruthy(); 120 | 121 | const iter2 = parser.write('"}'); 122 | expect(iter2.next().value).toMatchObject({ field: 'value"}' }); 123 | expect(iter2.next().done).toBeTruthy(); 124 | 125 | expect(parser.isLocked()).toBeFalsy(); 126 | expect(parser.hasPending()).toBeFalsy(); 127 | }); 128 | 129 | // We don't expect arrays, can be added as needed 130 | test.skip('can parse valid JSON array', () => { 131 | const parser = new IncrementalJSONParser(); 132 | 133 | const jsonArray = '["element1", "element2", "element3"]'; 134 | const iter = parser.write(jsonArray); 135 | 136 | const next = iter.next(); 137 | expect(next.value).toMatchObject(['element1', 'element2', 'element3']); 138 | expect(next.done).toBeFalsy(); 139 | 140 | expect(iter.next().done).toBeTruthy(); 141 | 142 | expect(parser.isLocked()).toBeFalsy(); 143 | expect(parser.hasPending()).toBeFalsy(); 144 | }); 145 | 146 | test('handles escaped sequences in a string correctly', () => { 147 | const parser = new IncrementalJSONParser(); 148 | const jsonObject = 149 | '{"data": "This is a \\"string\\" with escaped quotes and \\\\ double backslashes"}'; 150 | const iter = parser.write(jsonObject); 151 | const next = iter.next(); 152 | expect(next.value).toMatchObject({ 153 | data: 'This is a "string" with escaped quotes and \\ double backslashes', 154 | }); 155 | expect(next.done).toBeFalsy(); 156 | expect(iter.next().done).toBeTruthy(); 157 | expect(parser.isLocked()).toBeFalsy(); 158 | expect(parser.hasPending()).toBeFalsy(); 159 | }); 160 | --------------------------------------------------------------------------------