├── .eslintrc ├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .npmrc ├── LICENSE ├── README.md ├── build.config.ts ├── bun.lockb ├── docs ├── logo-dark.svg └── logo-light.svg ├── examples ├── cf-proxy │ ├── .editorconfig │ ├── .gitignore │ ├── .prettierrc │ ├── bun.lockb │ ├── package-lock.json │ ├── package.json │ ├── src │ │ └── worker.ts │ ├── tsconfig.json │ └── wrangler.toml ├── cf-workers │ ├── .editorconfig │ ├── .gitignore │ ├── .prettierrc │ ├── package-lock.json │ ├── package.json │ ├── src │ │ └── worker.ts │ ├── tsconfig.json │ └── wrangler.toml ├── embedding-comparison.ts ├── nuxt │ ├── .gitignore │ ├── .npmrc │ ├── README.md │ ├── app.vue │ ├── bun.lockb │ ├── nuxt.config.ts │ ├── package.json │ ├── public │ │ └── favicon.ico │ ├── server │ │ └── tsconfig.json │ └── tsconfig.json ├── simple-proxy-handler.ts ├── simple-script.ts └── use-model-schema.ts ├── package.json ├── src ├── assets │ └── price │ │ ├── anthropic.json │ │ └── openai.json ├── cursive.ts ├── function.ts ├── index.ts ├── pricing.ts ├── proxy.ts ├── schema.ts ├── stream.ts ├── types.ts ├── usage │ ├── anthropic.ts │ └── openai.ts ├── util.ts └── vendor │ ├── anthropic.ts │ ├── google.ts │ ├── index.ts │ └── openai.ts ├── test └── index.test.ts └── tsconfig.json /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@antfu", 3 | "rules": { 4 | "@typescript-eslint/indent": ["error", 4], 5 | "no-console": "off" 6 | }, 7 | "ignorePatterns": [ 8 | "dist/", 9 | "node_modules/", 10 | "examples/" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | lint: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | 18 | - name: Install pnpm 19 | uses: pnpm/action-setup@v2 20 | 21 | - name: Set node 22 | uses: actions/setup-node@v3 23 | with: 24 | node-version: 18.x 25 | cache: pnpm 26 | 27 | - name: Setup 28 | run: npm i -g @antfu/ni 29 | 30 | - name: Install 31 | run: nci 32 | 33 | - name: Lint 34 | run: nr lint 35 | 36 | typecheck: 37 | runs-on: ubuntu-latest 38 | steps: 39 | - uses: actions/checkout@v3 40 | 41 | - name: Install pnpm 42 | uses: pnpm/action-setup@v2 43 | 44 | - name: Set node 45 | uses: actions/setup-node@v3 46 | with: 47 | node-version: 18.x 48 | cache: pnpm 49 | 50 | - name: Setup 51 | run: npm i -g @antfu/ni 52 | 53 | - name: Install 54 | run: nci 55 | 56 | - name: Typecheck 57 | run: nr typecheck 58 | 59 | test: 60 | runs-on: ${{ matrix.os }} 61 | 62 | strategy: 63 | matrix: 64 | node: [16.x, 18.x] 65 | os: [ubuntu-latest, windows-latest, macos-latest] 66 | fail-fast: false 67 | 68 | steps: 69 | - uses: actions/checkout@v3 70 | 71 | - name: Install pnpm 72 | uses: pnpm/action-setup@v2 73 | 74 | - name: Set node ${{ matrix.node }} 75 | uses: actions/setup-node@v3 76 | with: 77 | node-version: ${{ matrix.node }} 78 | cache: pnpm 79 | 80 | - name: Setup 81 | run: npm i -g @antfu/ni 82 | 83 | - name: Install 84 | run: nci 85 | 86 | - name: Build 87 | run: nr build 88 | 89 | - name: Test 90 | run: nr test 91 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | permissions: 4 | contents: write 5 | 6 | on: 7 | push: 8 | tags: 9 | - 'v*' 10 | 11 | jobs: 12 | release: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v3 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: Install pnpm 20 | uses: pnpm/action-setup@v2 21 | 22 | - name: Set node 23 | uses: actions/setup-node@v3 24 | with: 25 | node-version: 18.x 26 | cache: pnpm 27 | 28 | - run: npx changelogithub 29 | env: 30 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | .DS_Store 3 | .idea 4 | *.log 5 | *.tgz 6 | coverage 7 | dist 8 | lib-cov 9 | logs 10 | node_modules 11 | temp 12 | playground/** -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | ignore-workspace-root-check=true 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Henrique Cunha 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Logo](/docs/logo-dark.svg#gh-dark-mode-only) 2 | ![Logo](/docs/logo-light.svg#gh-light-mode-only) 3 | 4 | Cursive is a universal and intuitive framework for interacting with LLMs. 5 | 6 | It works in any JavaScript runtime and has a heavy focus on extensibility and developer experience. 7 | 8 | ## ■ Highlights 9 | ✦ **Compatible** - Cursive works in any runtime, including the browser, Node.js, Deno, Bun and Cloudflare Workers. Through [WindowAI](https://windowai.io), users can securely bring their own credentials, provider, and model to completions. 10 | 11 | ✦ **Extensible** - You can easily hook into any part of a completion life cycle. Be it to log, cache, or modify the results. 12 | 13 | ✦ **Functions** - Easily describe functions that the LLM can use along with its definition, with any model (currently supporting GPT-4, GPT-3.5, Claude 2, and Claude Instant) 14 | 15 | ✦ **Universal** - Cursive's goal is to bridge as many capabilities between different models as possible. Ultimately, this means that with a single interface, you can allow your users to choose any model. 16 | 17 | ✦ **Informative** - Cursive comes with built-in token usage and costs calculations, as accurate as possible. 18 | 19 | ✦ **Reliable** - Cursive comes with automatic retry and model expanding upon exceeding context length. Which you can always configure. 20 | 21 | ## ■ Quickstart 22 | 1. Install. 23 | 24 | ```bash 25 | npm i cursive 26 | ``` 27 | 28 | 2. Start using. 29 | 30 | ```ts 31 | import { useCursive } from 'cursive' 32 | 33 | const cursive = useCursive({ 34 | openAI: { 35 | apiKey: 'sk-xxxx' 36 | } 37 | }) 38 | 39 | const { answer } = await cursive.ask({ 40 | prompt: 'What is the meaning of life?', 41 | }) 42 | ``` 43 | 44 | ## ■ Usage 45 | ### Conversation 46 | Chaining a conversation is easy with `cursive`. You can pass any of the options you're used to with OpenAI's API. 47 | 48 | ```ts 49 | const resA = await cursive.ask({ 50 | prompt: 'Give me a good name for a gecko.', 51 | model: 'gpt-4', 52 | maxTokens: 16, 53 | }) 54 | 55 | console.log(resA.answer) // Zephyr 56 | 57 | const resB = await resA.conversation.ask({ 58 | prompt: 'How would you say it in Portuguese?' 59 | }) 60 | 61 | console.log(resB.answer) // Zéfiro 62 | ``` 63 | ### Streaming 64 | Streaming is also supported, and we also keep track of the tokens for you! 65 | ```ts 66 | const result = await cursive.ask({ 67 | prompt: 'Count to 10', 68 | stream: true, 69 | onToken(partial) { 70 | console.log(partial.content) 71 | } 72 | }) 73 | 74 | console.log(result.usage.totalTokens) // 40 75 | ``` 76 | 77 | ### Functions 78 | You can use `Type` to define and describe functions, along side with their execution code. 79 | This is powered by the [**`typebox`**](https://github.com/sinclairzx81/typebox) library. 80 | ```ts 81 | import { Type, createFunction, useCursive } from 'cursive' 82 | 83 | const cursive = useCursive({ 84 | openAI: { 85 | apiKey: 'sk-xxxx' 86 | } 87 | }) 88 | 89 | const sum = createFunction({ 90 | name: 'sum', 91 | description: 'Sums two numbers', 92 | parameters: { 93 | a: Type.Number({ description: 'Number A' }), 94 | b: Type.Number({ description: 'Number B' }), 95 | }, 96 | async execute({ a, b }) { 97 | return a + b 98 | }, 99 | }) 100 | 101 | const { answer } = await cursive.ask({ 102 | prompt: 'What is the sum of 232 and 243?', 103 | functions: [sum], 104 | }) 105 | 106 | console.log(answer) // The sum of 232 and 243 is 475. 107 | ``` 108 | 109 | The functions' result will automatically be fed into the conversation and another completion will be made. If you want to prevent this, you can add `pause` to your function definition. 110 | 111 | ```ts 112 | const createCharacter = createFunction({ 113 | name: 'createCharacter', 114 | description: 'Creates a character', 115 | parameters: { 116 | name: Type.String({ description: 'The name of the character' }), 117 | age: Type.Number({ description: 'The age of the character' }), 118 | hairColor: Type.StringEnum(['black', 'brown', 'blonde', 'red', 'white'], { description: 'The hair color of the character' }), 119 | }, 120 | pause: true, 121 | async execute({ name, age, hairColor }) { 122 | return { name, age, hairColor } 123 | }, 124 | }) 125 | 126 | const { functionResult } = await cursive.ask({ 127 | prompt: 'Create a character named John who is 23 years old.', 128 | functions: [createCharacter], 129 | }) 130 | 131 | console.log(functionResult) // { name: 'John', age: 23 } 132 | ``` 133 | 134 | If you're on a `0.x.x` version, you can check here for the [old documentation](https://github.com/meistrari/cursive/tree/v0.12.2). 135 | 136 | ### Hooks 137 | You can hook into any part of the completion life cycle. 138 | ```ts 139 | cursive.on('completion:after', (result) => { 140 | console.log(result.cost.total) 141 | console.log(result.usage.total_tokens) 142 | }) 143 | 144 | cursive.on('completion:error', (error) => { 145 | console.log(error) 146 | }) 147 | 148 | cursive.ask({ 149 | prompt: 'Can androids dream of electric sheep?', 150 | }) 151 | 152 | // 0.0002185 153 | // 113 154 | ``` 155 | 156 | ### Embedding 157 | You can create embeddings pretty easily with `cursive`. 158 | ```ts 159 | const embedding = await cursive.embed('This should be a document.') 160 | ``` 161 | This will support different types of documents and integrations pretty soon. 162 | 163 | ### Reliability 164 | Cursive comes with automatic retry with backoff upon failing completions, and model expanding upon exceeding context length -- which means that it tries again with a model with a bigger context length when it fails by running out of it. 165 | 166 | You can configure this behavior by passing the `retry` and `expand` options to `useCursive`. 167 | 168 | ```ts 169 | const cursive = useCursive({ 170 | maxRetries: 5, // 0 disables it completely 171 | expand: { 172 | enable: true, 173 | defaultsTo: 'gpt-3.5-turbo-16k', 174 | modelMapping: { 175 | 'gpt-3.5-turbo': 'gpt-3.5-turbo-16k', 176 | 'gpt-4': 'claude-2', 177 | }, 178 | }, 179 | allowWindowAI: true, 180 | countUsage: false, // When disabled doesn't load and execute token counting and price estimates 181 | }) 182 | ``` 183 | 184 | ## ■ Examples 185 | 186 | - **[Nuxt ⇢ Simple Application](https://github.com/meistrari/cursive/blob/main/examples/nuxt)** 187 | - **[Cloudflare Workers ⇢ Simple Edge API](https://github.com/meistrari/cursive/blob/main/examples/cf-workers)** 188 | 189 | ## ■ Roadmap 190 | 191 | ### Vendor support 192 | - [x] Anthropic 193 | - [ ] Cohere (works on browser through WindowAI) 194 | - [ ] Azure OpenAI models 195 | - [ ] Huggingface (works on browser through WindowAI) 196 | - [ ] Replicate (works on browser through WindowAI) 197 | 198 | 199 | ## ■ Credits 200 | 201 | Thanks to [**@disjukr**](https://github.com/disjukr) for transferring the `cursive` npm package name to us! -------------------------------------------------------------------------------- /build.config.ts: -------------------------------------------------------------------------------- 1 | import { defineBuildConfig } from 'unbuild' 2 | 3 | export default defineBuildConfig({ 4 | entries: [ 5 | 'src/index', 6 | ], 7 | declaration: true, 8 | clean: true, 9 | rollup: { 10 | emitCJS: true, 11 | }, 12 | failOnWarn: false 13 | }) 14 | -------------------------------------------------------------------------------- /bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meistrari/cursive/2d9bd34e34229133793c4d62ce155a39e141a16e/bun.lockb -------------------------------------------------------------------------------- /docs/logo-dark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /docs/logo-light.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /examples/cf-proxy/.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = tab 6 | tab_width = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.yml] 13 | indent_style = space 14 | -------------------------------------------------------------------------------- /examples/cf-proxy/.gitignore: -------------------------------------------------------------------------------- 1 | .dev.vars 2 | .wrangler -------------------------------------------------------------------------------- /examples/cf-proxy/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "singleQuote": true, 4 | "semi": true, 5 | "useTabs": true 6 | } 7 | -------------------------------------------------------------------------------- /examples/cf-proxy/bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meistrari/cursive/2d9bd34e34229133793c4d62ce155a39e141a16e/examples/cf-proxy/bun.lockb -------------------------------------------------------------------------------- /examples/cf-proxy/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudflare-workers", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "deploy": "wrangler publish", 7 | "start": "wrangler dev" 8 | }, 9 | "devDependencies": { 10 | "@cloudflare/workers-types": "^4.20230419.0", 11 | "typescript": "^5.0.4", 12 | "wrangler": "^3.83.0" 13 | }, 14 | "dependencies": {} 15 | } 16 | -------------------------------------------------------------------------------- /examples/cf-proxy/src/worker.ts: -------------------------------------------------------------------------------- 1 | import { resguard } from 'resguard' 2 | import { createCursiveProxy } from '../../../src/index' 3 | 4 | export interface Env { 5 | OPENAI_API_KEY: string 6 | ANTHROPIC_API_KEY: string 7 | } 8 | 9 | export default { 10 | async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { 11 | const proxy = createCursiveProxy({ 12 | openAI: { apiKey: env.OPENAI_API_KEY }, 13 | anthropic: { apiKey: env.ANTHROPIC_API_KEY }, 14 | stream: { encodeValues: true }, 15 | countUsage: true 16 | }) 17 | 18 | proxy.on('query:after', (query) => { 19 | console.log(query?.cost) 20 | }) 21 | 22 | const body = await resguard(request.json()) 23 | 24 | if(body.error) { 25 | return new Response(JSON.stringify({ 26 | error: true, 27 | })) 28 | } 29 | 30 | const response = await proxy.handle(body.data) 31 | 32 | if (body.data.stream) { 33 | const init = { 34 | status: 200, 35 | statusText: 'ok', 36 | headers: new Headers({ 37 | 'Content-Type': 'text/event-stream', 38 | 'Cache-Control': 'no-cache', 39 | 'Connection': 'keep-alive', 40 | 'Access-Control-Allow-Origin': '*', 41 | 'Access-Control-Allow-Headers': 'Origin, X-Requested-With, Content-Type, Accept', 42 | }), 43 | } 44 | 45 | return new Response(response as ReadableStream, init) 46 | } else { 47 | return new Response(JSON.stringify(response)) 48 | } 49 | 50 | }, 51 | } 52 | -------------------------------------------------------------------------------- /examples/cf-proxy/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 4 | 5 | /* Projects */ 6 | // "incremental": true, /* Enable incremental compilation */ 7 | // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ 8 | // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ 9 | // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ 10 | // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ 11 | // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ 12 | 13 | /* Language and Environment */ 14 | "target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, 15 | "lib": ["es2021"] /* Specify a set of bundled library declaration files that describe the target runtime environment. */, 16 | "jsx": "react" /* Specify what JSX code is generated. */, 17 | // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ 18 | // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ 19 | // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ 20 | // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ 21 | // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ 22 | // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ 23 | // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ 24 | // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ 25 | 26 | /* Modules */ 27 | "module": "es2022" /* Specify what module code is generated. */, 28 | // "rootDir": "./", /* Specify the root folder within your source files. */ 29 | "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, 30 | // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ 31 | // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ 32 | // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ 33 | // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ 34 | "types": ["@cloudflare/workers-types"] /* Specify type package names to be included without being referenced in a source file. */, 35 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 36 | "resolveJsonModule": true /* Enable importing .json files */, 37 | // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ 38 | 39 | /* JavaScript Support */ 40 | "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */, 41 | "checkJs": false /* Enable error reporting in type-checked JavaScript files. */, 42 | // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ 43 | 44 | /* Emit */ 45 | // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ 46 | // "declarationMap": true, /* Create sourcemaps for d.ts files. */ 47 | // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ 48 | // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ 49 | // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ 50 | // "outDir": "./", /* Specify an output folder for all emitted files. */ 51 | // "removeComments": true, /* Disable emitting comments. */ 52 | "noEmit": true /* Disable emitting files from a compilation. */, 53 | // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ 54 | // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ 55 | // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ 56 | // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ 57 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 58 | // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ 59 | // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ 60 | // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ 61 | // "newLine": "crlf", /* Set the newline character for emitting files. */ 62 | // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ 63 | // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ 64 | // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ 65 | // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ 66 | // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ 67 | // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ 68 | 69 | /* Interop Constraints */ 70 | "isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */, 71 | "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */, 72 | // "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */, 73 | // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ 74 | "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, 75 | 76 | /* Type Checking */ 77 | "strict": true /* Enable all strict type-checking options. */, 78 | // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ 79 | // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ 80 | // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ 81 | // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ 82 | // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ 83 | // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ 84 | // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ 85 | // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ 86 | // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ 87 | // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ 88 | // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ 89 | // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ 90 | // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ 91 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ 92 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ 93 | // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ 94 | // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ 95 | // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ 96 | 97 | /* Completeness */ 98 | // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ 99 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /examples/cf-proxy/wrangler.toml: -------------------------------------------------------------------------------- 1 | name = "cursive-proxy" 2 | main = "src/worker.ts" 3 | compatibility_date = "2023-07-17" 4 | -------------------------------------------------------------------------------- /examples/cf-workers/.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = tab 6 | tab_width = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.yml] 13 | indent_style = space 14 | -------------------------------------------------------------------------------- /examples/cf-workers/.gitignore: -------------------------------------------------------------------------------- 1 | .dev.vars -------------------------------------------------------------------------------- /examples/cf-workers/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "singleQuote": true, 4 | "semi": true, 5 | "useTabs": true 6 | } 7 | -------------------------------------------------------------------------------- /examples/cf-workers/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudflare-workers", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "deploy": "wrangler publish", 7 | "start": "wrangler dev" 8 | }, 9 | "devDependencies": { 10 | "@cloudflare/workers-types": "^4.20230419.0", 11 | "typescript": "^5.0.4", 12 | "wrangler": "^3.83.0" 13 | } 14 | } -------------------------------------------------------------------------------- /examples/cf-workers/src/worker.ts: -------------------------------------------------------------------------------- 1 | import { useCursive } from '../../../src/index' 2 | 3 | export interface Env { 4 | OPENAI_API_KEY: string 5 | } 6 | 7 | export default { 8 | async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { 9 | const cursive = useCursive({ openAI: { apiKey: env.OPENAI_API_KEY } }) 10 | const { answer } = await cursive.ask({ 11 | prompt: 'Generate a random hello world message in a random language!', 12 | temperature: 1, 13 | maxTokens: 16, 14 | }) 15 | return new Response(answer) 16 | }, 17 | } 18 | -------------------------------------------------------------------------------- /examples/cf-workers/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Visit https://aka.ms/tsconfig.json to read more about this file */ 4 | 5 | /* Projects */ 6 | // "incremental": true, /* Enable incremental compilation */ 7 | // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ 8 | // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ 9 | // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ 10 | // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ 11 | // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ 12 | 13 | /* Language and Environment */ 14 | "target": "es2021" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, 15 | "lib": ["es2021"] /* Specify a set of bundled library declaration files that describe the target runtime environment. */, 16 | "jsx": "react" /* Specify what JSX code is generated. */, 17 | // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ 18 | // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ 19 | // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ 20 | // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ 21 | // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ 22 | // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ 23 | // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ 24 | // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ 25 | 26 | /* Modules */ 27 | "module": "es2022" /* Specify what module code is generated. */, 28 | // "rootDir": "./", /* Specify the root folder within your source files. */ 29 | "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, 30 | // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ 31 | // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ 32 | // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ 33 | // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ 34 | "types": ["@cloudflare/workers-types"] /* Specify type package names to be included without being referenced in a source file. */, 35 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 36 | "resolveJsonModule": true /* Enable importing .json files */, 37 | // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ 38 | 39 | /* JavaScript Support */ 40 | "allowJs": true /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */, 41 | "checkJs": false /* Enable error reporting in type-checked JavaScript files. */, 42 | // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ 43 | 44 | /* Emit */ 45 | // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ 46 | // "declarationMap": true, /* Create sourcemaps for d.ts files. */ 47 | // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ 48 | // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ 49 | // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ 50 | // "outDir": "./", /* Specify an output folder for all emitted files. */ 51 | // "removeComments": true, /* Disable emitting comments. */ 52 | "noEmit": true /* Disable emitting files from a compilation. */, 53 | // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ 54 | // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ 55 | // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ 56 | // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ 57 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 58 | // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ 59 | // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ 60 | // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ 61 | // "newLine": "crlf", /* Set the newline character for emitting files. */ 62 | // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ 63 | // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ 64 | // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ 65 | // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ 66 | // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ 67 | // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ 68 | 69 | /* Interop Constraints */ 70 | "isolatedModules": true /* Ensure that each file can be safely transpiled without relying on other imports. */, 71 | "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */, 72 | // "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */, 73 | // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ 74 | "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, 75 | 76 | /* Type Checking */ 77 | "strict": true /* Enable all strict type-checking options. */, 78 | // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ 79 | // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ 80 | // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ 81 | // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ 82 | // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ 83 | // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ 84 | // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ 85 | // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ 86 | // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ 87 | // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ 88 | // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ 89 | // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ 90 | // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ 91 | // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ 92 | // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ 93 | // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ 94 | // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ 95 | // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ 96 | 97 | /* Completeness */ 98 | // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ 99 | "skipLibCheck": true /* Skip type checking all .d.ts files. */ 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /examples/cf-workers/wrangler.toml: -------------------------------------------------------------------------------- 1 | name = "cloudflare-workers" 2 | main = "src/worker.ts" 3 | compatibility_date = "2023-07-21" 4 | -------------------------------------------------------------------------------- /examples/embedding-comparison.ts: -------------------------------------------------------------------------------- 1 | import { useCursive } from '../src/index' 2 | 3 | const cursive = useCursive() 4 | 5 | for (const _ of [1,1,1,1,1]) { 6 | const textA = 'Olá' 7 | const embeddingA = await cursive.embed(textA) 8 | 9 | const textB = 'Olá' 10 | const embeddingB = await cursive.embed(textB) 11 | 12 | const howSimilar = similarity(embeddingA, embeddingB) 13 | 14 | console.log({ 15 | textA, 16 | textB, 17 | howSimilar 18 | }) 19 | } 20 | 21 | function similarity(a: number[], b: number[]) { 22 | const dotProduct = a.reduce((acc, cur, i) => acc + cur * b[i], 0) 23 | const normA = Math.sqrt(a.reduce((acc, cur) => acc + cur * cur, 0)) 24 | const normB = Math.sqrt(b.reduce((acc, cur) => acc + cur * cur, 0)) 25 | return dotProduct / (normA * normB) 26 | } 27 | -------------------------------------------------------------------------------- /examples/nuxt/.gitignore: -------------------------------------------------------------------------------- 1 | # Nuxt dev/build outputs 2 | .output 3 | .nuxt 4 | .nitro 5 | .cache 6 | dist 7 | 8 | # Node dependencies 9 | node_modules 10 | 11 | # Logs 12 | logs 13 | *.log 14 | 15 | # Misc 16 | .DS_Store 17 | .fleet 18 | .idea 19 | 20 | # Local env files 21 | .env 22 | .env.* 23 | !.env.example 24 | -------------------------------------------------------------------------------- /examples/nuxt/.npmrc: -------------------------------------------------------------------------------- 1 | shamefully-hoist=true 2 | strict-peer-dependencies=false 3 | -------------------------------------------------------------------------------- /examples/nuxt/README.md: -------------------------------------------------------------------------------- 1 | # Nuxt 3 Minimal Starter 2 | 3 | Look at the [Nuxt 3 documentation](https://nuxt.com/docs/getting-started/introduction) to learn more. 4 | 5 | ## Setup 6 | 7 | Make sure to install the dependencies: 8 | 9 | ```bash 10 | # npm 11 | npm install 12 | 13 | # pnpm 14 | pnpm install 15 | 16 | # yarn 17 | yarn install 18 | ``` 19 | 20 | ## Development Server 21 | 22 | Start the development server on `http://localhost:3000`: 23 | 24 | ```bash 25 | # npm 26 | npm run dev 27 | 28 | # pnpm 29 | pnpm run dev 30 | 31 | # yarn 32 | yarn dev 33 | ``` 34 | 35 | ## Production 36 | 37 | Build the application for production: 38 | 39 | ```bash 40 | # npm 41 | npm run build 42 | 43 | # pnpm 44 | pnpm run build 45 | 46 | # yarn 47 | yarn build 48 | ``` 49 | 50 | Locally preview production build: 51 | 52 | ```bash 53 | # npm 54 | npm run preview 55 | 56 | # pnpm 57 | pnpm run preview 58 | 59 | # yarn 60 | yarn preview 61 | ``` 62 | 63 | Check out the [deployment documentation](https://nuxt.com/docs/getting-started/deployment) for more information. 64 | -------------------------------------------------------------------------------- /examples/nuxt/app.vue: -------------------------------------------------------------------------------- 1 | 20 | 21 | 26 | -------------------------------------------------------------------------------- /examples/nuxt/bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meistrari/cursive/2d9bd34e34229133793c4d62ce155a39e141a16e/examples/nuxt/bun.lockb -------------------------------------------------------------------------------- /examples/nuxt/nuxt.config.ts: -------------------------------------------------------------------------------- 1 | // https://nuxt.com/docs/api/configuration/nuxt-config 2 | export default defineNuxtConfig({ 3 | devtools: { enabled: true }, 4 | $development: { 5 | build: { 6 | transpile: ['@web-std/stream'], 7 | }, 8 | }, 9 | }) 10 | -------------------------------------------------------------------------------- /examples/nuxt/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nuxt-app", 3 | "private": true, 4 | "scripts": { 5 | "build": "nuxt build", 6 | "dev": "nuxt dev", 7 | "generate": "nuxt generate", 8 | "preview": "nuxt preview", 9 | "postinstall": "nuxt prepare" 10 | }, 11 | "devDependencies": { 12 | "@nuxt/devtools": "latest", 13 | "@types/node": "^18.16.19", 14 | "nuxt": "^3.6.5" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /examples/nuxt/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/meistrari/cursive/2d9bd34e34229133793c4d62ce155a39e141a16e/examples/nuxt/public/favicon.ico -------------------------------------------------------------------------------- /examples/nuxt/server/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../.nuxt/tsconfig.server.json" 3 | } 4 | -------------------------------------------------------------------------------- /examples/nuxt/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | // https://nuxt.com/docs/guide/concepts/typescript 3 | "extends": "./.nuxt/tsconfig.json" 4 | } 5 | -------------------------------------------------------------------------------- /examples/simple-proxy-handler.ts: -------------------------------------------------------------------------------- 1 | import { createCursiveProxy, Type as t } from "../src" 2 | 3 | const person = t.Object({ 4 | name: t.String(), 5 | age: t.Number(), 6 | }, { 7 | title: 'Person', 8 | description: 'A person object' 9 | }) 10 | 11 | const proxy = createCursiveProxy({ countUsage: true }) 12 | 13 | const response = await proxy.handle({ 14 | messages: [{ 'role': 'user', 'content': 'WHATS THE WEATHER IN SAN FRANCISCO?' }], 15 | model: 'gpt-3.5-turbo-16k', 16 | functions: [ 17 | { 18 | name: 'getWeather', 19 | description: 'returns the weather in a specific city', 20 | parameters: { 21 | properties: { 22 | city: { 23 | type: 'string', 24 | description: 'the city to get the weather for', 25 | } 26 | }, 27 | type: 'object', 28 | required: ['city'] 29 | } 30 | } 31 | ] 32 | }) 33 | 34 | console.log(response) 35 | 36 | if ('choices' in response) { 37 | console.log(response.choices.at(0)) 38 | } 39 | 40 | -------------------------------------------------------------------------------- /examples/simple-script.ts: -------------------------------------------------------------------------------- 1 | import { useCursive } from '../src/index' 2 | 3 | const cursive = useCursive({ countUsage: true }) 4 | 5 | const { answer, usage, error } = await cursive.ask({ 6 | prompt: 'Tell me a short short joke', 7 | model: 'claude-3-sonnet-20240229', 8 | }) 9 | 10 | console.log({ 11 | answer, 12 | usage, 13 | error: error?.details 14 | }) 15 | -------------------------------------------------------------------------------- /examples/use-model-schema.ts: -------------------------------------------------------------------------------- 1 | import { useCursive, Type as t } from '../src/index' 2 | 3 | const cursive = useCursive() 4 | 5 | const schema = t.Object({ 6 | name: t.String({ description: 'The name of the person' }), 7 | age: t.Number({ description: 'The age of the person' }), 8 | pets: t.Array( 9 | t.Object({ 10 | name: t.String({ description: 'The name of the pet' }), 11 | age: t.Number({ description: 'The age of the pet' }), 12 | type: t.String({ description: 'The type of the pet' }), 13 | }), 14 | { description: 'The pets of the person' }, 15 | ), 16 | }, { 17 | title: 'Person', 18 | description: 'A person object' 19 | }) 20 | 21 | const { answer } = await cursive.ask({ 22 | schema, 23 | model: 'claude-2', 24 | prompt: 'Create a person named John with 2 pets named Fluffy and Fido', 25 | }) 26 | 27 | console.log(answer) 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cursive", 3 | "type": "module", 4 | "version": "2.6.0", 5 | "description": "", 6 | "author": "Henrique Cunha ", 7 | "license": "MIT", 8 | "homepage": "https://github.com/meistrari/cursive-gpt#readme", 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/meistrari/cursive-gpt.git" 12 | }, 13 | "bugs": "https://github.com/meistrari/cursive-gpt/issues", 14 | "keywords": [], 15 | "sideEffects": false, 16 | "exports": { 17 | ".": { 18 | "types": "./dist/index.d.ts", 19 | "require": "./dist/index.cjs", 20 | "import": "./dist/index.mjs" 21 | } 22 | }, 23 | "main": "dist/index.mjs", 24 | "module": "dist/index.mjs", 25 | "types": "dist/index.d.ts", 26 | "files": [ 27 | "dist" 28 | ], 29 | "scripts": { 30 | "build": "unbuild", 31 | "dev": "unbuild --stub", 32 | "lint": "eslint .", 33 | "prepublishOnly": "nr build", 34 | "release": "bumpp && npm publish", 35 | "start": "esno src/index.ts", 36 | "test": "vitest", 37 | "typecheck": "tsc --noEmit" 38 | }, 39 | "devDependencies": { 40 | "@antfu/eslint-config": "^0.39.8", 41 | "@antfu/ni": "^0.21.12", 42 | "@antfu/utils": "^0.7.10", 43 | "@nuxt/kit": "^3.13.1", 44 | "@types/node": "^18.19.50", 45 | "@types/whatwg-streams": "^3.2.1", 46 | "bumpp": "^9.5.2", 47 | "bun-types": "1.0.4-canary.20231003T140149", 48 | "eslint": "^8.57.0", 49 | "esno": "^0.16.3", 50 | "haxx": "^0.7.0", 51 | "jsonrepair": "^3.8.0", 52 | "lint-staged": "^13.3.0", 53 | "node-fetch": "^3.3.2", 54 | "rimraf": "^5.0.10", 55 | "simple-git-hooks": "^2.11.1", 56 | "typescript": "^5.5.4", 57 | "unbuild": "^1.2.1", 58 | "vite": "^4.5.3", 59 | "vitest": "^0.31.4", 60 | "window.ai": "^0.2.4" 61 | }, 62 | "dependencies": { 63 | "@sinclair/typebox": "^0.31.28", 64 | "@web-std/stream": "^1.0.3", 65 | "eventsource-parser": "^1.1.2", 66 | "hookable": "^5.5.3", 67 | "hotscript": "^1.0.13", 68 | "isomorphic-streams": "^1.0.3", 69 | "ofetch": "^1.3.4", 70 | "openai-edge": "^1.2.2", 71 | "resguard": "^1.4.3", 72 | "unenv": "^1.10.0", 73 | "unitoken": "^0.0.1" 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/assets/price/anthropic.json: -------------------------------------------------------------------------------- 1 | { 2 | "claude-instant": { 3 | "completion": 0.0008, 4 | "prompt": 0.0024 5 | }, 6 | "claude-2": { 7 | "completion": 0.008, 8 | "prompt": 0.024 9 | }, 10 | "claude-3-haiku-20240307": { 11 | "completion": 0.00025, 12 | "prompt": 0.00125 13 | }, 14 | "claude-3-sonnet-20240229": { 15 | "completion": 0.003, 16 | "prompt": 0.015 17 | }, 18 | "claude-3-opus-20240229": { 19 | "completion": 0.015, 20 | "prompt": 0.075 21 | }, 22 | "version": "2024-03-15" 23 | } -------------------------------------------------------------------------------- /src/assets/price/openai.json: -------------------------------------------------------------------------------- 1 | { 2 | "gpt-4": { 3 | "completion": 0.06, 4 | "prompt": 0.03 5 | }, 6 | "gpt-4-32k": { 7 | "completion": 0.12, 8 | "prompt": 0.06 9 | }, 10 | "gpt-3.5-turbo": { 11 | "completion": 0.002, 12 | "prompt": 0.0015 13 | }, 14 | "gpt-3.5-turbo-16k": { 15 | "completion": 0.004, 16 | "prompt": 0.003 17 | }, 18 | "version": "2023-06-13" 19 | } -------------------------------------------------------------------------------- /src/cursive.ts: -------------------------------------------------------------------------------- 1 | import type { ChatMessage, CompletionOptions, MessageOutput, WindowAI } from 'window.ai' 2 | 3 | import type { ChatCompletionRequestMessage, ChatCompletionRequestMessageFunctionCall, CreateChatCompletionRequest, CreateChatCompletionResponse, CreateChatCompletionResponseChoicesInner } from 'openai-edge' 4 | import { resguard } from 'resguard' 5 | import type { Hookable } from 'hookable' 6 | import { createDebugger, createHooks } from 'hookable' 7 | import type { CursiveAnswerResult, CursiveAskCost, CursiveAskOnToken, CursiveAskOptions, CursiveAskOptionsWithPrompt, CursiveAskUsage, CursiveFunction, CursiveFunctionSchema, CursiveHook, CursiveHooks, CursiveSetupOptions } from './types' 8 | import { CursiveError, CursiveErrorCode } from './types' 9 | import type { IfNull } from './util' 10 | import { randomId, sleep, toSnake } from './util' 11 | import { resolveAnthropicPricing, resolveOpenAIPricing } from './pricing' 12 | import { createOpenAIClient, processOpenAIStream } from './vendor/openai' 13 | import { resolveVendorFromModel } from './vendor' 14 | import { createAnthropicClient, getAnthropicFunctionCallDirectives, processAnthropicStream } from './vendor/anthropic' 15 | import { getOpenAIUsage } from './usage/openai' 16 | import { getAnthropicUsage } from './usage/anthropic' 17 | import { schemaToFunction } from './schema' 18 | import { TSchema } from '@sinclair/typebox' 19 | import { jsonrepair } from 'jsonrepair' 20 | 21 | declare let window: { ai: WindowAI } 22 | 23 | export class Cursive { 24 | public _hooks: Hookable 25 | public _vendor: { 26 | openai: ReturnType 27 | anthropic: ReturnType 28 | } 29 | 30 | public options: CursiveSetupOptions 31 | 32 | private _usingWindowAI = false 33 | private _debugger: { close: () => void } 34 | private _ready = false 35 | 36 | constructor(options: CursiveSetupOptions = { 37 | countUsage: true, 38 | }) { 39 | this._hooks = createHooks() 40 | this._vendor = { 41 | openai: createOpenAIClient({ apiKey: options?.openAI?.apiKey || ('process' in globalThis && process.env.OPENAI_API_KEY) }), 42 | anthropic: createAnthropicClient({ apiKey: options?.anthropic?.apiKey || ('process' in globalThis && process.env.ANTHROPIC_API_KEY) }), 43 | } 44 | this.options = options 45 | 46 | if (options.debug) 47 | this._debugger = createDebugger(this._hooks, { tag: 'cursive' }) 48 | 49 | if (options.allowWindowAI === undefined || options.allowWindowAI === true) { 50 | if (typeof window !== 'undefined') { 51 | // Wait for the window.ai to be available, for a maximum of half a second 52 | const start = Date.now() 53 | const interval = setInterval(() => { 54 | if (window.ai) { 55 | clearInterval(interval) 56 | this._usingWindowAI = true 57 | this._ready = true 58 | if (options.debug) 59 | console.log('[cursive] Using WindowAI') 60 | } 61 | else if (Date.now() - start > 500) { 62 | clearInterval(interval) 63 | this._ready = true 64 | } 65 | }, 100) 66 | } 67 | else { 68 | this._ready = true 69 | } 70 | } 71 | else { 72 | this._ready = true 73 | } 74 | } 75 | 76 | private _readyCheck() { 77 | return new Promise((resolve) => { 78 | let tries = 0 79 | const interval = setInterval(() => { 80 | if (this._ready || ++tries > 80) { 81 | clearInterval(interval) 82 | resolve(null) 83 | } 84 | }, 10) 85 | }) 86 | } 87 | 88 | on(event: H, callback: CursiveHooks[H]) { 89 | this._hooks.hook(event, callback as any) 90 | } 91 | 92 | async ask( 93 | options: CursiveAskOptions, 94 | ): Promise> { 97 | await this._readyCheck() 98 | const result = await buildAnswer(options, this) 99 | 100 | if (result.error) { 101 | return new CursiveAnswer({ 102 | result: null, 103 | error: result.error, 104 | }) 105 | } 106 | 107 | const newMessages = [ 108 | ...result.messages, 109 | { role: 'assistant', content: result.answer } as const, 110 | ] 111 | 112 | return new CursiveAnswer({ 113 | result, 114 | error: null, 115 | messages: newMessages, 116 | cursive: this, 117 | }) 118 | } 119 | 120 | async embed(content: string) { 121 | await this._readyCheck() 122 | const options = { 123 | model: 'text-embedding-ada-002', 124 | input: content, 125 | } 126 | await this._hooks.callHook('embedding:before', options) 127 | const start = Date.now() 128 | const response = await this._vendor.openai.createEmbedding(options) 129 | 130 | const data = await response.json() 131 | 132 | if (data.error) { 133 | const error = new CursiveError(data.error.message, data.error, CursiveErrorCode.EmbeddingError) 134 | await this._hooks.callHook('embedding:error', error, Date.now() - start) 135 | await this._hooks.callHook('embedding:after', null, error, Date.now() - start) 136 | throw error 137 | } 138 | const result = { 139 | embedding: data.data[0].embedding, 140 | } 141 | await this._hooks.callHook('embedding:success', result, Date.now() - start) 142 | await this._hooks.callHook('embedding:after', result, null, Date.now() - start) 143 | 144 | return result.embedding as number[] 145 | } 146 | } 147 | 148 | export class CursiveConversation { 149 | public _cursive: Cursive 150 | public messages: ChatCompletionRequestMessage[] = [] 151 | 152 | constructor(messages: ChatCompletionRequestMessage[]) { 153 | this.messages = messages 154 | } 155 | 156 | async ask(options: CursiveAskOptionsWithPrompt): Promise> { 157 | const { prompt, ...rest } = options 158 | const resolvedOptions = { 159 | ...(rest as any), 160 | messages: [ 161 | ...this.messages, 162 | { role: 'user', content: prompt }, 163 | ], 164 | } 165 | 166 | const result = await buildAnswer(resolvedOptions, this._cursive) 167 | 168 | if (result.error) { 169 | return new CursiveAnswer({ 170 | result: null, 171 | error: result.error, 172 | }) 173 | } 174 | 175 | const newMessages = [ 176 | ...result.messages, 177 | { role: 'assistant', content: result.answer } as const, 178 | ] 179 | 180 | return new CursiveAnswer({ 181 | result, 182 | error: null, 183 | messages: newMessages, 184 | cursive: this._cursive, 185 | }) 186 | } 187 | } 188 | 189 | export class CursiveAnswer { 190 | public choices: IfNull 191 | public id: IfNull 192 | public model: IfNull 193 | public usage: IfNull 194 | public cost: IfNull 195 | public error: E 196 | public functionResult?: IfNull 197 | /** 198 | * The text from the answer of the last choice 199 | */ 200 | public answer: IfNull 201 | /** 202 | * A conversation instance with all the messages so far, including this one 203 | */ 204 | public conversation: IfNull 205 | 206 | constructor(options: { 207 | result: any | null 208 | error: E 209 | messages?: ChatCompletionRequestMessage[] 210 | cursive?: Cursive 211 | }) { 212 | if (options.error) { 213 | this.error = options.error 214 | this.choices = null 215 | this.id = null 216 | this.model = null 217 | this.usage = null 218 | this.cost = null 219 | this.answer = null 220 | this.conversation = null 221 | this.functionResult = null 222 | } 223 | else { 224 | this.error = null 225 | this.choices = options.result.choices 226 | this.id = options.result.id 227 | this.model = options.result.model 228 | this.usage = options.result.usage 229 | this.cost = options.result.cost 230 | this.answer = options.result.answer 231 | this.functionResult = options.result.functionResult 232 | const conversation = new CursiveConversation(options.messages) as any 233 | conversation._cursive = options.cursive 234 | this.conversation = conversation 235 | } 236 | } 237 | } 238 | 239 | export function useCursive(options: CursiveSetupOptions = {}) { 240 | return new Cursive(options) 241 | } 242 | 243 | function resolveOptions(options: CursiveAskOptions) { 244 | const { 245 | functions: functionsRaw = [], 246 | messages = [], 247 | model = 'gpt-3.5-turbo-0613', 248 | systemMessage, 249 | prompt, 250 | schema, 251 | functionCall, 252 | abortSignal: __, 253 | ...rest 254 | } = options 255 | 256 | // TODO: Add support for function call resolving 257 | const vendor = resolveVendorFromModel(model) 258 | let resolvedSystemMessage = systemMessage || '' 259 | 260 | if (schema) { 261 | const resolvedSchemaFunction = schemaToFunction(schema) 262 | 263 | functionsRaw.push({ 264 | definition: async (args) => args, 265 | pause: true, 266 | schema: resolvedSchemaFunction, 267 | } as any) 268 | } 269 | 270 | const functions = resolveFunctionList(functionsRaw) 271 | const resolvedFunctionCall = resolveFunctionCall(functionCall, schemaToFunction(schema)) 272 | 273 | if (vendor === 'anthropic' && functions.length > 0) 274 | resolvedSystemMessage = `${systemMessage || ''}\n\n${getAnthropicFunctionCallDirectives(functions,)}` 275 | 276 | const hasSystemMessage = messages.some(message => message.role === 'system') 277 | 278 | let filteredMessages = messages 279 | if (hasSystemMessage && resolvedSystemMessage) 280 | filteredMessages = messages.filter(message => message.role !== 'system') 281 | 282 | const queryMessages = [ 283 | resolvedSystemMessage && { role: 'system', content: resolvedSystemMessage }, 284 | ...filteredMessages, 285 | prompt && { role: 'user', content: prompt }, 286 | ].filter(Boolean) as ChatCompletionRequestMessage[] 287 | 288 | 289 | const payload: CreateChatCompletionRequest = { 290 | ...toSnake(rest), 291 | model, 292 | messages: queryMessages, 293 | function_call: resolvedFunctionCall, 294 | } 295 | 296 | const resolvedOptions = { 297 | ...rest, 298 | model, 299 | schema, 300 | functions, 301 | functionCall: resolvedFunctionCall, 302 | messages: queryMessages, 303 | } 304 | 305 | return { payload, resolvedOptions } 306 | } 307 | 308 | function resolveFunctionCall(functionCall: any, schema?: any) { 309 | if (schema) { 310 | return { name: schema.name } 311 | } 312 | 313 | if (functionCall) { 314 | if (typeof functionCall === 'string') { 315 | return functionCall 316 | } else { 317 | return { name: functionCall.schema.name } 318 | } 319 | } else { 320 | return undefined 321 | } 322 | } 323 | 324 | async function createCompletion(context: { 325 | payload: CreateChatCompletionRequest 326 | cursive: Cursive 327 | abortSignal?: AbortSignal 328 | onToken?: CursiveAskOnToken 329 | }) { 330 | const { payload, abortSignal } = context 331 | await context.cursive._hooks.callHook('completion:before', payload) 332 | const start = Date.now() 333 | 334 | let data: CreateChatCompletionResponse & { cost: CursiveAskCost; error: any } 335 | const vendor = resolveVendorFromModel(payload.model) 336 | 337 | // TODO: Improve the completion creation based on model to vendor matching 338 | // For now this will do 339 | // @ts-expect-error - We're using a private property here 340 | if (context.cursive._usingWindowAI) { 341 | const resolvedModel = vendor ? `${vendor}/${payload.model}` : payload.model 342 | 343 | const options: CompletionOptions = { 344 | maxTokens: payload.max_tokens, 345 | model: resolvedModel, 346 | numOutputs: payload.n, 347 | stopSequences: payload.stop as string[], 348 | temperature: payload.temperature, 349 | } 350 | 351 | if (payload.stream && context.onToken) { 352 | options.onStreamResult = (result) => { 353 | const resultResolved = result as MessageOutput 354 | context.onToken({ content: resultResolved.message.content, functionCall: null }) 355 | } 356 | } 357 | 358 | const response = await window.ai.generateText({ 359 | messages: payload.messages as ChatMessage[], 360 | }, options) as MessageOutput[] 361 | data = {} as any 362 | data.choices = response.map(choice => ({ 363 | message: choice.message, 364 | })) 365 | data.model = payload.model 366 | data.id = randomId() 367 | data.usage = { 368 | prompt_tokens: null, 369 | completion_tokens: null, 370 | total_tokens: null, 371 | } as any 372 | 373 | if (context.cursive.options.countUsage) { 374 | const content = data.choices.map(choice => choice.message.content).join('') 375 | if (vendor === 'openai') { 376 | data.usage.prompt_tokens = await getOpenAIUsage(context.payload.messages) 377 | data.usage.completion_tokens = await getOpenAIUsage(content) 378 | } 379 | 380 | else if (vendor === 'anthropic') { 381 | data.usage.prompt_tokens = await getAnthropicUsage(context.payload.messages) 382 | data.usage.completion_tokens = await getAnthropicUsage(content) 383 | } 384 | 385 | else { 386 | // TODO: Create better estimations for other vendors 387 | data.usage.prompt_tokens = await getOpenAIUsage(context.payload.messages) 388 | data.usage.completion_tokens = await getOpenAIUsage(content) 389 | data.usage.total_tokens = data.usage.completion_tokens + data.usage.prompt_tokens 390 | } 391 | } 392 | } 393 | else { 394 | if (vendor === 'openai') { 395 | const response = await context.cursive._vendor.openai.createChatCompletion({ ...payload }, abortSignal) 396 | if (payload.stream) { 397 | data = await processOpenAIStream({ ...context, response }) 398 | const content = data.choices.map(choice => choice.message.content).join('') 399 | if (context.cursive.options.countUsage) { 400 | data.usage.completion_tokens = await getOpenAIUsage(content) 401 | data.usage.total_tokens = data.usage.completion_tokens + data.usage.prompt_tokens 402 | } 403 | } 404 | else { 405 | data = await response.json() 406 | } 407 | } 408 | else if (vendor === 'anthropic') { 409 | const response = await context.cursive._vendor.anthropic({ ...payload }, abortSignal) 410 | if (payload.stream) { 411 | data = await processAnthropicStream({ ...context, response }) 412 | } 413 | else { 414 | const responseData = await response.json() 415 | 416 | if (responseData.error) 417 | throw new CursiveError(responseData.error.message, responseData.error, CursiveErrorCode.CompletionError) 418 | 419 | data = { 420 | choices: [{ message: { content: responseData.content[0].text.trimStart() } }], 421 | model: payload.model, 422 | id: randomId(), 423 | usage: {} as any, 424 | } as any 425 | } 426 | 427 | // We check for function call in the completion 428 | const hasFunctionCallRegex = /([^<]+)<\/function-call>/ 429 | const functionCallMatches = data.choices[0].message.content.match(hasFunctionCallRegex) 430 | 431 | if (functionCallMatches) { 432 | const functionCall = JSON.parse(jsonrepair(functionCallMatches[1].trim())) 433 | data.choices[0].message.function_call = { 434 | name: functionCall.name, 435 | arguments: JSON.stringify(functionCall.arguments), 436 | } 437 | } 438 | if (context.cursive.options.countUsage) { 439 | data.usage.prompt_tokens = await getAnthropicUsage(context.payload.messages) 440 | data.usage.completion_tokens = await getAnthropicUsage(data.choices[0].message.content) 441 | data.usage.total_tokens = data.usage.completion_tokens + data.usage.prompt_tokens 442 | } 443 | // We check for answers in the completion 444 | const hasAnswerRegex = /([^<]+)<\/cursive-answer>/ 445 | const answerMatches = data.choices[0].message.content.match(hasAnswerRegex) 446 | if (answerMatches) { 447 | const answer = answerMatches[1].trim() 448 | data.choices[0].message.content = answer 449 | } 450 | } 451 | } 452 | if (context.cursive.options.countUsage && data.usage) { 453 | if (vendor === 'openai') { 454 | data.cost = resolveOpenAIPricing({ 455 | completionTokens: data.usage.completion_tokens, 456 | promptTokens: data.usage.prompt_tokens, 457 | totalTokens: data.usage.total_tokens, 458 | }, data.model) 459 | } 460 | else if (vendor === 'anthropic') { 461 | data.cost = resolveAnthropicPricing({ 462 | completionTokens: data.usage.completion_tokens, 463 | promptTokens: data.usage.prompt_tokens, 464 | totalTokens: data.usage.total_tokens, 465 | }, data.model) 466 | } 467 | } 468 | else { 469 | data.cost = null 470 | } 471 | 472 | const end = Date.now() 473 | 474 | if (data.error) { 475 | const error = new CursiveError(data.error.message, data.error, CursiveErrorCode.CompletionError) 476 | await context.cursive._hooks.callHook('completion:error', error, end - start) 477 | await context.cursive._hooks.callHook('completion:after', null, error, end - start) 478 | throw error 479 | } 480 | 481 | await context.cursive._hooks.callHook('completion:success', data, end - start) 482 | await context.cursive._hooks.callHook('completion:after', data, null, end - start) 483 | 484 | return data as CreateChatCompletionResponse & { cost: CursiveAskCost } 485 | } 486 | 487 | async function askModel( 488 | options: CursiveAskOptions, 489 | cursive: Cursive, 490 | ): Promise<{ 491 | answer: CreateChatCompletionResponse & { functionResult?: any; cost: CursiveAskCost } 492 | messages: ChatCompletionRequestMessage[] 493 | }> { 494 | await cursive._hooks.callHook('query:before', options as any) 495 | 496 | const { payload, resolvedOptions } = resolveOptions(options as any) 497 | const functions = resolveFunctionList(options.functions || []) 498 | 499 | // Check if the user passed a schema, if so, we add it to the functions list 500 | if (typeof options.functionCall !== 'string' && options.functionCall?.schema) 501 | resolvedOptions.functions.push(options.functionCall) 502 | 503 | const functionSchemas = resolvedOptions.functions.map(({ schema }) => schema) 504 | 505 | if (functionSchemas.length > 0) 506 | payload.functions = functionSchemas 507 | 508 | let completion = await resguard(createCompletion({ 509 | payload, 510 | cursive, 511 | onToken: options.onToken, 512 | abortSignal: options.abortSignal, 513 | }), CursiveError) 514 | 515 | if (completion.error) { 516 | if (!completion.error?.details) 517 | throw new CursiveError(`Unknown error: ${completion.error.message}`, completion.error, CursiveErrorCode.UnknownError, completion.error.stack) 518 | 519 | const cause = completion.error.details.code || completion.error.details.type 520 | if (cause === 'context_length_exceeded') { 521 | if (!cursive.options.expand || cursive.options.expand?.enabled === true) { 522 | const defaultModel = cursive.options?.expand?.defaultsTo || 'gpt-3.5-turbo-16k' 523 | const modelMapping = cursive.options?.expand?.modelMapping || {} 524 | const resolvedModel = modelMapping[options.model] || defaultModel 525 | completion = await resguard( 526 | createCompletion({ 527 | payload: { ...payload, model: resolvedModel }, 528 | cursive, 529 | onToken: options.onToken, 530 | abortSignal: options.abortSignal, 531 | }), 532 | CursiveError, 533 | ) 534 | } 535 | } 536 | 537 | else if (cause === 'invalid_request_error') { 538 | throw new CursiveError('Invalid request', completion.error.details, CursiveErrorCode.InvalidRequestError) 539 | } 540 | 541 | // TODO: Handle other errors 542 | if (completion.error) { 543 | // TODO: Add a more comprehensive retry strategy 544 | for (let i = 0; i < cursive.options.maxRetries; i++) { 545 | completion = await resguard(createCompletion({ 546 | payload, 547 | cursive, 548 | onToken: options.onToken, 549 | abortSignal: options.abortSignal, 550 | }), CursiveError) 551 | 552 | if (!completion.error) { 553 | if (i > 3) 554 | await sleep(1000 * (i - 3) * 2) 555 | break 556 | } 557 | } 558 | } 559 | } 560 | 561 | if (completion.error) { 562 | const error = new CursiveError('Error while completing request', completion.error.details, CursiveErrorCode.CompletionError) 563 | await cursive._hooks.callHook('query:error', error) 564 | await cursive._hooks.callHook('query:after', null, error) 565 | throw error 566 | } 567 | 568 | // Check if any of the choices has a function call and if an schema is passed 569 | const hasFunctionCall = completion.data.choices.some(choice => choice.message.function_call) 570 | if (hasFunctionCall && resolvedOptions.schema) { 571 | completion.data.choices = completion.data.choices.map((choice) => { 572 | const args = resguard(() => JSON.parse(jsonrepair(choice.message.function_call.arguments || '{}')), SyntaxError) 573 | return { 574 | ...choice, 575 | message: { 576 | content: args.data, 577 | role: 'assistant' 578 | } 579 | } 580 | }) 581 | 582 | return { 583 | answer: { 584 | ...completion.data, 585 | functionResult: completion.data.choices.at(-1).message.content, 586 | }, 587 | messages: payload.messages || [], 588 | } 589 | } 590 | 591 | if (completion.data?.choices[0].message?.function_call) { 592 | payload.messages.push({ 593 | role: 'assistant', 594 | function_call: completion.data.choices[0].message?.function_call, 595 | content: '', 596 | }) 597 | const functionCall = completion.data.choices[0].message?.function_call 598 | const functionDefinition = functions.find(({ schema }) => schema.name === functionCall.name) 599 | 600 | if (!functionDefinition) { 601 | return await askModel( 602 | { 603 | ...resolvedOptions as any, 604 | functionCall: 'none', 605 | messages: payload.messages, 606 | }, 607 | cursive, 608 | ) 609 | } 610 | 611 | const args = resguard(() => JSON.parse(jsonrepair(functionCall.arguments || '{}')), SyntaxError) 612 | const functionResult = await resguard(functionDefinition.definition(args.data)) 613 | 614 | if (functionResult.error) { 615 | throw new CursiveError( 616 | `Error while running function ${functionCall.name}`, 617 | functionResult.error, 618 | CursiveErrorCode.FunctionCallError, 619 | ) 620 | } 621 | 622 | const messages = payload.messages || [] 623 | 624 | messages.push({ 625 | role: 'function', 626 | name: functionCall.name, 627 | content: JSON.stringify(functionResult.data || ''), 628 | }) 629 | 630 | if (functionDefinition.pause) { 631 | return { 632 | answer: { 633 | ...completion.data, 634 | functionResult: functionResult.data, 635 | }, 636 | messages, 637 | } 638 | } 639 | else { 640 | return await askModel( 641 | { 642 | ...resolvedOptions as any, 643 | functions, 644 | messages, 645 | }, 646 | cursive, 647 | ) 648 | } 649 | } 650 | 651 | await cursive._hooks.callHook('query:after', completion.data, null) 652 | await cursive._hooks.callHook('query:success', completion.data) 653 | 654 | return { 655 | answer: completion.data, 656 | messages: payload.messages || [], 657 | } 658 | } 659 | 660 | async function buildAnswer( 661 | options: CursiveAskOptions, 662 | cursive: Cursive, 663 | ): Promise { 664 | const result = await resguard(askModel(options, cursive), CursiveError) 665 | 666 | 667 | if (result.error) { 668 | return { 669 | error: result.error, 670 | usage: null, 671 | model: options.model || 'gpt-3.5-turbo-16k', 672 | id: null, 673 | choices: null, 674 | functionResult: null, 675 | answer: null, 676 | messages: null, 677 | cost: null, 678 | } 679 | } 680 | else { 681 | const usage: CursiveAskUsage = { 682 | completionTokens: result.data.answer.usage!.completion_tokens, 683 | promptTokens: result.data.answer.usage!.prompt_tokens, 684 | totalTokens: result.data.answer.usage!.total_tokens, 685 | } 686 | 687 | let resolvedAnswer = result.data.answer.choices.at(-1).message.content 688 | 689 | if (options.schema) { 690 | const output = result.data.answer.functionResult 691 | // Validate the output against the schema 692 | resolvedAnswer = output 693 | } 694 | 695 | const newMessage = { 696 | error: null, 697 | model: result.data.answer.model, 698 | id: result.data.answer.id, 699 | usage, 700 | cost: result.data.answer.cost, 701 | choices: resolveChoices(result.data.answer.choices), 702 | functionResult: result.data.answer.functionResult || null, 703 | answer: resolvedAnswer, 704 | messages: result.data.messages, 705 | } 706 | 707 | return newMessage 708 | } 709 | } 710 | 711 | function resolveChoices(choices: CreateChatCompletionResponseChoicesInner[]) { 712 | return choices.map(choice => choice.message.content || choice.message.function_call) 713 | } 714 | 715 | function resolveFunctionList(functions: (CursiveFunction | CursiveFunctionSchema)[]) { 716 | return functions.map((functionDefinition) => { 717 | if ('schema' in functionDefinition) { 718 | if (!functionDefinition.definition) { 719 | functionDefinition.definition = async (args) => args; 720 | } 721 | return functionDefinition 722 | } 723 | else if ('name' in functionDefinition) { 724 | const fn: CursiveFunction = { 725 | schema: functionDefinition, 726 | pause: true, 727 | definition: async (args) => args, 728 | } 729 | return fn 730 | } 731 | return null 732 | }).filter(Boolean) 733 | } 734 | 735 | interface CursiveEnrichedAnswer { 736 | error: CursiveError | null 737 | usage: CursiveAskUsage 738 | model: string 739 | id: string 740 | choices: (string | ChatCompletionRequestMessageFunctionCall)[] 741 | functionResult: any 742 | answer: string 743 | messages: ChatCompletionRequestMessage[] 744 | cost: CursiveAskCost 745 | } 746 | -------------------------------------------------------------------------------- /src/function.ts: -------------------------------------------------------------------------------- 1 | import { type TProperties, Type } from '@sinclair/typebox' 2 | import type { CursiveCreateFunctionOptions, CursiveFunction } from './types' 3 | 4 | export function createFunction

(options: CursiveCreateFunctionOptions

): CursiveFunction { 5 | const parameters = Type.Object(options.parameters ?? {}) 6 | const { description, name } = options 7 | 8 | const resolvedSchema = { 9 | parameters: { 10 | properties: deepRemoveNonStringKeys(parameters.properties), 11 | required: parameters.required, 12 | type: 'object' as const, 13 | }, 14 | description, 15 | name, 16 | } 17 | 18 | return { 19 | schema: resolvedSchema, 20 | definition: options.execute, 21 | pause: options.pause, 22 | } 23 | } 24 | 25 | function deepRemoveNonStringKeys(obj: any) { 26 | const newObj: any = {} 27 | for (const key in obj) { 28 | if (typeof obj[key] === 'string') 29 | newObj[key] = obj[key] 30 | else if (Array.isArray(obj[key])) 31 | newObj[key] = obj[key] 32 | else if (typeof obj[key] === 'object') 33 | newObj[key] = deepRemoveNonStringKeys(obj[key]) 34 | } 35 | return newObj 36 | } 37 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { createFunction } from './function' 2 | export { useCursive } from './cursive' 3 | export * from './types' 4 | export { createCursiveProxy, type CursiveProxyRequest } from './proxy' 5 | -------------------------------------------------------------------------------- /src/pricing.ts: -------------------------------------------------------------------------------- 1 | import type { CursiveAskCost, CursiveAskUsage } from './types' 2 | import OpenAIPrices from './assets/price/openai.json' 3 | import AnthropicPrices from './assets/price/anthropic.json' 4 | 5 | export function resolveOpenAIPricing(usage: CursiveAskUsage, model: string) { 6 | const modelsAvailable = Object.keys(OpenAIPrices) 7 | const modelMatch = modelsAvailable.find(m => model.startsWith(m)) 8 | if (!modelMatch) 9 | throw new Error(`Unknown model ${model}`) 10 | 11 | const modelPrice = OpenAIPrices[modelMatch] 12 | const { completionTokens, promptTokens } = usage 13 | const completion = completionTokens * modelPrice.completion / 1000 14 | const prompt = promptTokens * modelPrice.prompt / 1000 15 | 16 | const cost: CursiveAskCost = { 17 | completion, 18 | prompt, 19 | total: completion + prompt, 20 | version: OpenAIPrices.version, 21 | } 22 | 23 | return cost 24 | } 25 | 26 | export function resolveAnthropicPricing(usage: CursiveAskUsage, model: string) { 27 | const modelsAvailable = Object.keys(AnthropicPrices) 28 | const modelMatch = modelsAvailable.find(m => model.startsWith(m)) 29 | if (!modelMatch) 30 | throw new Error(`Unknown model ${model}`) 31 | 32 | const modelPrice = AnthropicPrices[modelMatch] 33 | const { completionTokens, promptTokens } = usage 34 | const completion = completionTokens * modelPrice.completion / 1000 35 | const prompt = promptTokens * modelPrice.prompt / 1000 36 | 37 | const cost: CursiveAskCost = { 38 | completion, 39 | prompt, 40 | total: completion + prompt, 41 | version: AnthropicPrices.version, 42 | } 43 | 44 | return cost 45 | } 46 | -------------------------------------------------------------------------------- /src/proxy.ts: -------------------------------------------------------------------------------- 1 | import type { CreateChatCompletionResponse, ErrorResponse } from 'openai-edge' 2 | import { TransformStream } from '@web-std/stream' 3 | import type { TSchema } from '@sinclair/typebox' 4 | import type { CursiveAskOptions, CursiveAskOptionsWithMessages, CursiveHook, CursiveHooks, CursiveSetupOptions, CursiveStreamDelta } from './types' 5 | import { useCursive } from './cursive' 6 | import { randomId } from './util' 7 | 8 | interface CursiveProxyOptions { 9 | stream?: { 10 | encodeValues?: boolean 11 | } 12 | } 13 | 14 | export type CursiveProxyRequest = CursiveAskOptions & { 15 | schema?: any 16 | } 17 | 18 | type CursiveProxy = (request: R) => Promise | ErrorResponse> 19 | 20 | export function createCursiveProxy(options: CursiveSetupOptions & CursiveProxyOptions = {}) { 21 | const cursive = useCursive(options) 22 | 23 | const handle: CursiveProxy = async (request) => { 24 | if (!request.stream) 25 | return await handleRequest(request, cursive) 26 | else 27 | return await handleStreamRequest(request, cursive, options.stream) 28 | } 29 | 30 | function on(event: H, callback: CursiveHooks[H]) { 31 | return cursive.on(event, callback) 32 | } 33 | 34 | return { 35 | handle, 36 | on, 37 | } 38 | } 39 | 40 | async function handleRequest(request: CursiveAskOptions, cursive: ReturnType) { 41 | const answer = await cursive.ask( 42 | request, 43 | ) 44 | 45 | if (answer.error) { 46 | return { 47 | error: { 48 | message: answer.error.message, 49 | name: answer.error.name, 50 | cause: answer.error.cause, 51 | details: answer.error.details, 52 | stack: answer.error.stack, 53 | }, 54 | } 55 | } 56 | 57 | 58 | const mappedAnswer: CreateChatCompletionResponse & { 59 | usage?: CreateChatCompletionResponse['usage'] & { 60 | cost?: { 61 | completion_cost: number 62 | prompt_cost: number 63 | total_cost: number 64 | } 65 | } 66 | } = { 67 | choices: answer.choices.map((choice, index) => { 68 | const resolvedChoice = { 69 | finish_reason: 'stop', 70 | index, 71 | message: {} as any, 72 | } 73 | 74 | if (typeof choice === 'object' && ('name' in choice && 'arguments' in choice)) { 75 | resolvedChoice.message = { 76 | role: 'function', 77 | function_call: choice 78 | } 79 | } else { 80 | resolvedChoice.message = { 81 | role: 'user', 82 | content: choice 83 | } 84 | } 85 | 86 | return resolvedChoice 87 | }), 88 | created: Date.now(), 89 | id: answer.id, 90 | object: 'chat.completion', 91 | usage: null, 92 | model: request.model, 93 | } 94 | 95 | if (answer.usage) { 96 | mappedAnswer.usage = { 97 | prompt_tokens: answer.usage.promptTokens, 98 | completion_tokens: answer.usage.completionTokens, 99 | total_tokens: answer.usage.totalTokens, 100 | } 101 | 102 | if (answer.cost) { 103 | mappedAnswer.usage.cost = { 104 | completion_cost: answer.cost.completion, 105 | prompt_cost: answer.cost.prompt, 106 | total_cost: answer.cost.total, 107 | } 108 | } 109 | } 110 | 111 | return mappedAnswer 112 | } 113 | 114 | // Wraps the request in a async generator function 115 | async function handleStreamRequest(request: CursiveProxyRequest, cursive: ReturnType, options?: CursiveProxyOptions['stream']) { 116 | async function getAsyncIterator() { 117 | // Define a queue to store tokens 118 | const tokens = [] 119 | // Set the initial resolver to null 120 | let resolver = null 121 | 122 | // Initiate your request and pass the handler 123 | cursive.ask({ 124 | ...request as CursiveAskOptionsWithMessages, 125 | onToken: (token) => { 126 | // If the resolver exists, resolve it with a new promise and reset 127 | if (resolver) { 128 | const currentResolver = resolver 129 | // create a new promise and reset resolver 130 | resolver = null 131 | currentResolver({ value: token, done: false }) 132 | } 133 | else { 134 | // Otherwise, push the token into the queue 135 | tokens.push(token) 136 | } 137 | }, 138 | }).then(() => { 139 | // When the request is complete, resolve the last token 140 | if (resolver) { 141 | const currentResolver = resolver 142 | // create a new promise and reset resolver 143 | resolver = null 144 | currentResolver({ value: undefined, done: true }) 145 | } 146 | }) 147 | 148 | return { 149 | [Symbol.asyncIterator]() { 150 | return { 151 | // This is the iterator object 152 | next(): Promise> { 153 | if (tokens.length > 0) { 154 | const value = tokens.shift() 155 | if (value === null) 156 | return Promise.resolve({ value: undefined, done: true }) 157 | else 158 | return Promise.resolve({ value, done: false }) 159 | } 160 | 161 | // If no tokens queued, return a new promise 162 | return new Promise((resolve) => { 163 | // Set the resolver to resolve with the next token 164 | resolver = resolve 165 | }) 166 | }, 167 | } 168 | }, 169 | } 170 | } 171 | 172 | const iterableRequest = await getAsyncIterator() 173 | 174 | function asyncIteratorToReadableStream(options: { 175 | iterator: AsyncIterable 176 | transform?: (value: CursiveStreamDelta) => T 177 | onEnd?: () => E 178 | }) { 179 | const reader = options.iterator[Symbol.asyncIterator]() 180 | const { readable, writable } = new TransformStream() 181 | const writer = writable.getWriter() 182 | 183 | async function write() { 184 | const { done, value } = await reader.next() 185 | if (done) { 186 | if (options.onEnd) { 187 | const onEndValue = options.onEnd() 188 | if (Array.isArray(onEndValue)) { 189 | for (const v of onEndValue) 190 | writer.write(options.transform ? options.transform(v) : v) 191 | } 192 | } 193 | writer.close() 194 | } 195 | else { 196 | writer.write(options.transform ? options.transform(value) : value) 197 | write() 198 | } 199 | } 200 | 201 | write() 202 | return readable 203 | } 204 | 205 | let chunkData: string | any = { 206 | created: Date.now(), 207 | id: randomId(), 208 | model: request.model, 209 | object: 'chat.completion.chunk', 210 | choices: [], 211 | } 212 | 213 | const stopSymbol = Symbol('stop') 214 | 215 | const stream = asyncIteratorToReadableStream({ 216 | iterator: iterableRequest, 217 | transform: (delta) => { 218 | const isDone = (delta as any) === '[DONE]' 219 | const isStop = (delta as any) === stopSymbol 220 | 221 | if (isDone) { 222 | chunkData = '[DONE]' 223 | } 224 | else { 225 | chunkData.choices = [{ 226 | finish_reason: isStop ? 'stop' : delta.finishReason, 227 | delta: isStop ? {} : { content: delta.content }, 228 | index: delta.index, 229 | }] 230 | } 231 | 232 | if (options?.encodeValues) 233 | return new TextEncoder().encode(`data: ${JSON.stringify(chunkData, null, 4)}\n\n`) 234 | else 235 | return `data: ${JSON.stringify(chunkData, null, 4)}\n\n` 236 | }, 237 | onEnd: () => [stopSymbol, '[DONE]'], 238 | }) 239 | 240 | return stream 241 | } 242 | -------------------------------------------------------------------------------- /src/schema.ts: -------------------------------------------------------------------------------- 1 | import { TSchema } from "@sinclair/typebox" 2 | 3 | export function schemaToFunction(schema: TSchema | undefined) { 4 | if(!schema) 5 | return undefined 6 | 7 | // Remove all the typebox symbols 8 | const { title: name, description, ...resolvedSchema} = JSON.parse(JSON.stringify(schema)) 9 | 10 | const resolved = { 11 | name, 12 | description, 13 | parameters: resolvedSchema 14 | } 15 | 16 | return resolved 17 | } 18 | 19 | -------------------------------------------------------------------------------- /src/stream.ts: -------------------------------------------------------------------------------- 1 | import type { PassThrough } from 'node:stream' 2 | import type { EventSourceParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser' 3 | import { createParser } from 'eventsource-parser' 4 | import { ReadableStream, TransformStream } from '@web-std/stream' 5 | import { CursiveError, CursiveErrorCode } from './types' 6 | 7 | export async function getStream(res: Response): Promise { 8 | if (!res.ok) { 9 | const { error } = await res.json() 10 | throw new CursiveError( 11 | error.message, 12 | { 13 | ...error, 14 | }, 15 | CursiveErrorCode.CompletionError, 16 | ) 17 | } 18 | 19 | let stream: ReadableStream = res.body as any 20 | if (!stream.pipeThrough) { 21 | const passThrough = res.body as unknown as PassThrough 22 | stream = passThroughToReadableStream(passThrough) 23 | } 24 | return stream.pipeThrough ? stream.pipeThrough(eventTransformer()) : emptyStream() 25 | } 26 | 27 | function eventTransformer() { 28 | const textDecoder = new TextDecoder() 29 | let parser: EventSourceParser 30 | const parseEvent = createParseEvent() 31 | 32 | return new TransformStream({ 33 | async start(controller: any): Promise { 34 | parser = createParser((event: ParsedEvent | ReconnectInterval) => { 35 | if ( 36 | 'data' in event 37 | && event.type === 'event' 38 | && event.data === '[DONE]' 39 | ) 40 | return 41 | 42 | if ('data' in event && event.type === 'event') { 43 | const parsedEvent = parseEvent(event.data) 44 | if (parsedEvent) 45 | controller.enqueue(parsedEvent) 46 | } 47 | }) 48 | }, 49 | 50 | transform(chunk: any) { 51 | parser.feed(textDecoder.decode(chunk)) 52 | }, 53 | }) 54 | } 55 | 56 | function createParseEvent() { 57 | return (data: string) => JSON.parse(data) 58 | } 59 | 60 | function emptyStream() { 61 | return new ReadableStream({ 62 | start(controller: any) { 63 | controller.close() 64 | }, 65 | }) 66 | } 67 | 68 | export function createDecoder() { 69 | const textDecoder = new TextDecoder() 70 | 71 | return (chunk?: Uint8Array): string => { 72 | if (!chunk) 73 | return '' 74 | 75 | return textDecoder.decode(chunk) 76 | } 77 | } 78 | 79 | function passThroughToReadableStream(passThrough: PassThrough) { 80 | return new ReadableStream({ 81 | start(controller: any) { 82 | passThrough.on('data', (chunk) => { 83 | controller.enqueue(chunk) 84 | }) 85 | passThrough.on('end', () => { 86 | controller.close() 87 | }) 88 | passThrough.on('error', (err) => { 89 | controller.error(err) 90 | }) 91 | }, 92 | cancel() { 93 | passThrough.destroy() 94 | }, 95 | }) as unknown as ReadableStream 96 | } 97 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | import type { TProperties, TSchema } from '@sinclair/typebox' 2 | import type { ChatCompletionRequestMessage } from 'openai-edge' 3 | import type { CreateChatCompletionRequest, CreateChatCompletionResponse } from 'openai-edge' 4 | import { Type as TypeBox } from '@sinclair/typebox' 5 | import type { HookResult, ObjectWithNullValues, Override } from './util' 6 | import type { CursiveAnswer } from './cursive' 7 | 8 | export type CursiveAvailableModels = 9 | /* OpenAI */ 'gpt-3.5-turbo' | 'gpt-4' 10 | /* Anthropic */ | 'claude-instant-1' | 'claude-2' 11 | 12 | /* Allow any */ | (string & {}) 13 | 14 | export type InferredFunctionParameters = { 15 | [K in keyof T]: T[K]['static'] 16 | } 17 | 18 | export interface CursiveCreateFunctionOptions

{ 19 | name: string 20 | description: string 21 | parameters?: P 22 | execute(parameters: InferredFunctionParameters

): Promise 23 | pause?: boolean 24 | } 25 | 26 | export interface CursiveFunction { 27 | schema: CursiveFunctionSchema 28 | definition: (parameters: Record) => Promise 29 | pause?: boolean 30 | } 31 | 32 | export interface CursiveFunctionSchema { 33 | parameters: { 34 | type: 'object' 35 | properties: any 36 | required: string[] 37 | } 38 | description: string 39 | name: string 40 | } 41 | 42 | export interface CursiveSetupOptions { 43 | openAI?: { 44 | apiKey: string 45 | host?: string 46 | } 47 | anthropic?: { 48 | apiKey: string 49 | } 50 | maxRetries?: number 51 | expand?: { 52 | enabled?: boolean 53 | defaultsTo?: string 54 | modelMapping?: Record 55 | } 56 | debug?: boolean 57 | /** 58 | * Allows for the usage of WindowAI 59 | */ 60 | allowWindowAI?: boolean 61 | /** 62 | * Count usage and pricing for each completion 63 | */ 64 | countUsage?: boolean 65 | } 66 | 67 | export enum CursiveErrorCode { 68 | FunctionCallError = 'function_call_error', 69 | CompletionError = 'completion_error', 70 | InvalidRequestError = 'invalid_request_error', 71 | EmbeddingError = 'embedding_error', 72 | UnknownError = 'unknown_error', 73 | } 74 | export class CursiveError extends Error { 75 | constructor(public message: string, public details?: any, public code?: CursiveErrorCode, stack?: any) { 76 | super(message) 77 | this.name = 'CursiveError' 78 | this.message = message 79 | this.details = details 80 | this.code = code 81 | this.stack = stack 82 | } 83 | } 84 | 85 | export type CursiveStreamDelta = { 86 | functionCall: { name: string; arguments: '' } | { name: null; arguments: string } 87 | content: null 88 | index?: number 89 | finishReason?: string 90 | } 91 | | { 92 | content: string 93 | functionCall: null 94 | index?: number 95 | finishReason?: string 96 | } 97 | export type CursiveAskOnToken = (delta: CursiveStreamDelta) => void | Promise 98 | 99 | interface CursiveAskOptionsBase { 100 | model?: CursiveAvailableModels 101 | systemMessage?: string 102 | functions?: CursiveFunction[] | CursiveFunctionSchema[] 103 | functionCall?: string | CursiveFunction 104 | onToken?: CursiveAskOnToken 105 | maxTokens?: number 106 | stop?: string[] 107 | temperature?: number 108 | topP?: number 109 | presencePenalty?: number 110 | frequencyPenalty?: number 111 | bestOf?: number 112 | n?: number 113 | logitBias?: Record 114 | user?: string 115 | stream?: boolean 116 | schema?: T 117 | abortSignal?: AbortSignal 118 | } 119 | 120 | export interface CursiveAskOptionsWithMessages extends CursiveAskOptionsBase { 121 | messages: ChatCompletionRequestMessage[] 122 | prompt?: never 123 | } 124 | 125 | export interface CursiveAskOptionsWithPrompt extends CursiveAskOptionsBase { 126 | prompt: string 127 | messages?: never 128 | } 129 | 130 | export type CursiveAskOptions = CursiveAskOptionsWithMessages | CursiveAskOptionsWithPrompt 131 | 132 | export interface CursiveAskUsage { 133 | completionTokens: number 134 | promptTokens: number 135 | totalTokens: number 136 | } 137 | 138 | export interface CursiveAskCost { 139 | completion: number 140 | prompt: number 141 | total: number 142 | version: string 143 | } 144 | 145 | export type CursiveAnswerSuccess = CursiveAnswer 146 | export type CursiveAnswerError = Override>, { error: CursiveError }> 147 | export type CursiveAnswerResult = CursiveAnswerSuccess | CursiveAnswerError 148 | 149 | export interface CursiveAskErrorResult { 150 | choices: null 151 | id: null 152 | model: string 153 | usage: null 154 | cost: null 155 | answer: null 156 | conversation: null 157 | error: CursiveError 158 | } 159 | 160 | // export type CursiveAskResult = CursiveAnswer | 161 | type ChatCompletionWithCost = CreateChatCompletionResponse & { cost: CursiveAskCost } 162 | 163 | export interface CursiveHooks { 164 | 'query:before': (options: CursiveAskOptions) => HookResult 165 | 'query:after': (result: ChatCompletionWithCost | null, error: CursiveError | null) => HookResult 166 | 'query:error': (error: CursiveError) => HookResult 167 | 'query:success': (result: ChatCompletionWithCost) => HookResult 168 | 'completion:before': (options: CreateChatCompletionRequest) => HookResult 169 | 'completion:after': (result: ChatCompletionWithCost | null, error: CursiveError | null, duration: number) => HookResult 170 | 'completion:error': (error: CursiveError, duration: number) => HookResult 171 | 'completion:success': (result: ChatCompletionWithCost, duration: number) => HookResult 172 | 'embedding:before': (options: { model: string; input: string }) => HookResult 173 | 'embedding:after': (result: { embedding: number[] } | null, error: CursiveError | null, duration: number) => HookResult 174 | 'embedding:error': (error: CursiveError, duration: number) => HookResult 175 | 'embedding:success': (result: { embedding: number[] }, duration: number) => HookResult 176 | } 177 | 178 | export type CursiveHook = keyof CursiveHooks 179 | 180 | const Nullable = (schema: T) => TypeBox.Union([schema, TypeBox.Null()]) 181 | function StringEnum(values: [...T], options?: { description?: string }) { 182 | return TypeBox.Unsafe({ 183 | type: 'string', 184 | enum: values, 185 | ...options, 186 | }) 187 | } 188 | 189 | // @ts-expect-error Overriding method 190 | TypeBox.StringEnum = StringEnum 191 | // @ts-expect-error Overriding method 192 | TypeBox.Nullable = Nullable 193 | 194 | export const Type = TypeBox as any as typeof TypeBox & { 195 | Nullable: typeof Nullable 196 | StringEnum: typeof StringEnum 197 | } 198 | -------------------------------------------------------------------------------- /src/usage/anthropic.ts: -------------------------------------------------------------------------------- 1 | import type { ChatCompletionRequestMessage } from 'openai-edge' 2 | 3 | export async function getAnthropicUsage(content: string | ChatCompletionRequestMessage[]) { 4 | const { AnthropicEncoder } = await import('unitoken') 5 | 6 | if (typeof content === 'string') 7 | return AnthropicEncoder.encode(content).length 8 | 9 | const mappedContent = content.map((message) => { 10 | const { content, role } = message 11 | if (role === 'system') { 12 | return ` 13 | Human: ${content} 14 | 15 | Assistant: Ok. 16 | ` 17 | } 18 | return `${role}: ${content}` 19 | }).join('\n\n') 20 | 21 | return AnthropicEncoder.encode(mappedContent).length 22 | } 23 | -------------------------------------------------------------------------------- /src/usage/openai.ts: -------------------------------------------------------------------------------- 1 | import type { ChatCompletionFunctions, ChatCompletionRequestMessage } from 'openai-edge' 2 | 3 | export async function getOpenAIUsage(contentOrMessageList: string | ChatCompletionRequestMessage[]) { 4 | const { OpenAIEncoder } = await import('unitoken') 5 | 6 | if (typeof contentOrMessageList === 'string') 7 | return OpenAIEncoder.encode(contentOrMessageList).length 8 | 9 | const tokens = { 10 | perMessage: 0, 11 | perName: 0, 12 | } 13 | 14 | tokens.perMessage = 3 15 | tokens.perName = 1 16 | 17 | let tokenCount = 3 18 | for (const message of contentOrMessageList) { 19 | tokenCount += tokens.perMessage 20 | for (const key in message) { 21 | if (key === 'name') 22 | tokenCount += tokens.perName 23 | 24 | let value = (message as any)[key] 25 | if (typeof value === 'object') 26 | value = JSON.stringify(value) 27 | 28 | if (value === null || value === undefined) 29 | continue 30 | tokenCount += OpenAIEncoder.encode(value).length 31 | } 32 | } 33 | 34 | return tokenCount 35 | } 36 | 37 | export async function getTokenCountFromFunctions(functions: ChatCompletionFunctions[]) { 38 | const { OpenAIEncoder} = await import('unitoken') 39 | 40 | let tokenCount = 3 41 | for (const fn of functions) { 42 | let functionTokens = OpenAIEncoder.encode(fn.name).length 43 | functionTokens += OpenAIEncoder.encode(fn.description).length 44 | 45 | if (fn.parameters?.properties) { 46 | const properties = fn.parameters.properties 47 | for (const key in properties) { 48 | functionTokens += OpenAIEncoder.encode(key).length 49 | const value = properties[key] 50 | for (const field in value) { 51 | if (['type', 'description'].includes(field)) { 52 | functionTokens += 2 53 | functionTokens += OpenAIEncoder.encode(value[field]).length 54 | } 55 | else if (field === 'enum') { 56 | functionTokens -= 3 57 | for (const enumValue in value[field]) { 58 | functionTokens += 3 59 | functionTokens += OpenAIEncoder.encode(enumValue).length 60 | } 61 | } 62 | } 63 | } 64 | functionTokens += 11 65 | } 66 | tokenCount += functionTokens 67 | } 68 | tokenCount += 12 69 | return tokenCount 70 | } 71 | -------------------------------------------------------------------------------- /src/util.ts: -------------------------------------------------------------------------------- 1 | export type HookResult = Promise | void 2 | 3 | export type Override = Pick> & U 4 | 5 | type CamelToSnakeCaseKey = S extends `${infer T}${infer U}` ? `${T extends Uppercase ? `_${Lowercase}` : T}${CamelToSnakeCaseKey}` : S 6 | export type CamelToSnakeCase = T extends Array 7 | ? Array> 8 | : T extends object 9 | ? { [K in keyof T as CamelToSnakeCaseKey]: CamelToSnakeCase } 10 | : T 11 | 12 | const lowercase = (w: string) => w.toLowerCase() 13 | const toSnakeString = (w: string) => w.split(/(?=[A-Z])/).map(lowercase).join('_') 14 | 15 | export function toSnake(source: T): CamelToSnakeCase { 16 | if (Array.isArray(source)) 17 | return source.map(toSnake) as any 18 | 19 | if (source && typeof source === 'object') { 20 | const target = {} as any 21 | for (const [key, value] of Object.entries(source)) { 22 | const newKey = toSnakeString(key) 23 | target[newKey] = toSnake(value) 24 | } 25 | return target 26 | } 27 | return source as CamelToSnakeCase 28 | } 29 | 30 | export function sleep(ms: number) { 31 | return new Promise(resolve => setTimeout(resolve, ms)) 32 | } 33 | 34 | // Makes every key of the object null 35 | export type ObjectWithNullValues> = { 36 | [K in keyof T]: null 37 | } 38 | 39 | // Override keys of T with keys of U 40 | export type IfNull = T extends null ? U : null 41 | 42 | export function randomId() { 43 | return Math.random().toString(36).substring(7) 44 | } 45 | 46 | export function trim(content: string) { 47 | const lines = content.split('\n') 48 | let minIndent = Number.POSITIVE_INFINITY 49 | for (const line of lines) { 50 | const indent = line.search(/\S/) 51 | if (indent !== -1) 52 | minIndent = Math.min(minIndent, indent) 53 | } 54 | 55 | content = '' 56 | for (const line of lines) 57 | content += `${line.slice(minIndent)}\n` 58 | 59 | return content.trim() 60 | } 61 | -------------------------------------------------------------------------------- /src/vendor/anthropic.ts: -------------------------------------------------------------------------------- 1 | import { ofetch } from 'ofetch' 2 | import type { ChatCompletionRequestMessage, CreateChatCompletionRequest } from 'openai-edge' 3 | import type { Cursive } from '../cursive' 4 | import type { CursiveAskOnToken, CursiveFunction } from '../types' 5 | import { getStream } from '../stream' 6 | import { getAnthropicUsage } from '../usage/anthropic' 7 | import { trim } from '../util' 8 | 9 | 10 | 11 | export function createAnthropicClient(options: { apiKey: string }) { 12 | const resolvedFetch = ofetch.native 13 | 14 | async function createCompletion(payload: CreateChatCompletionRequest, abortSignal?: AbortSignal) { 15 | const response = await resolvedFetch('https://api.anthropic.com/v1/messages', { 16 | method: 'POST', 17 | headers: { 18 | 'Content-Type': 'application/json', 19 | 'x-api-key': options.apiKey, 20 | }, 21 | body: JSON.stringify({ 22 | messages: payload.messages, 23 | max_tokens: payload.max_tokens || 4096, 24 | stop_sequences: payload.stop, 25 | temperature: payload.temperature, 26 | top_p: payload.top_p, 27 | model: payload.model, 28 | // TODO: Add top_k support 29 | // top_k: payload.top_k, 30 | stream: payload.stream, 31 | }), 32 | signal: abortSignal, 33 | }) 34 | return response 35 | } 36 | 37 | return createCompletion 38 | } 39 | 40 | function buildAnthropicInput(messages: ChatCompletionRequestMessage[]) { 41 | const roleMapping = { user: 'Human', assistant: 'Assistant' } 42 | const messagesWithPrefix: ChatCompletionRequestMessage[] = [ 43 | ...messages, 44 | { role: 'assistant', content: ' ' }, 45 | ] 46 | return messagesWithPrefix.map((message) => { 47 | const { content, role, function_call, name } = message 48 | if (role === 'system') { 49 | return [ 50 | 'Human:', 51 | content, 52 | '\nAssistant: Ok.', 53 | ].join('\n') 54 | } 55 | if (role === 'function') { 56 | return [ 57 | `Human: `, 58 | content, 59 | '', 60 | ].join('\n') 61 | } 62 | if (function_call) { 63 | return [ 64 | 'Assistant: ', 65 | JSON.stringify({ 66 | name: function_call.name, 67 | arguments: typeof function_call.arguments === 'string' ? function_call.arguments : JSON.stringify(function_call.arguments), 68 | }), 69 | '', 70 | ].join('\n') 71 | } 72 | return `${roleMapping[role]}: ${content}` 73 | }).join('\n\n') 74 | } 75 | 76 | export async function processAnthropicStream(context: { 77 | payload: CreateChatCompletionRequest 78 | cursive: Cursive 79 | abortSignal?: AbortSignal 80 | onToken?: CursiveAskOnToken 81 | response: Response 82 | }) { 83 | let data: any 84 | 85 | const reader = (await getStream(context.response)).getReader() 86 | data = { 87 | choices: [{ message: { content: '' } }], 88 | usage: { 89 | completion_tokens: 0, 90 | prompt_tokens: await getAnthropicUsage(context.payload.messages), 91 | }, 92 | model: context.payload.model, 93 | } 94 | 95 | while (true) { 96 | const { done, value } = await reader.read() 97 | 98 | if (done) 99 | break 100 | 101 | data = { 102 | ...data, 103 | id: value.id, 104 | } 105 | 106 | // The completion partial will come with a leading whitespace 107 | value.completion = value.completion.trimStart() 108 | 109 | // Check if theres any tag. The regex should allow for nested tags 110 | const functionCallTag = value.completion.match(/([\s\S]*?)(?=<\/function-call>|$)/g) 111 | let functionName = '' 112 | let functionArguments = '' 113 | if (functionCallTag) { 114 | // Remove starting and ending tags, even if the ending tag is partial or missing 115 | const functionCall = functionCallTag[0] 116 | .replace(/<\/?f?u?n?c?t?i?o?n?-?c?a?l?l?>?/g, '') 117 | .trim() 118 | .replace(/^\n|\n$/g, '') 119 | .trim() 120 | // Match the function name inside the JSON 121 | functionName = functionCall.match(/"name":\s*"(.+)"/)?.[1] 122 | functionArguments = functionCall.match(/"arguments":\s*(\{.+)\}?/s)?.[1] 123 | if (functionArguments) { 124 | // If theres unmatches } at the end, remove them 125 | const unmatchedBrackets = functionArguments.match(/(\{|\})/g) 126 | if (unmatchedBrackets.length % 2) 127 | functionArguments = functionArguments.trim().replace(/\}$/, '') 128 | 129 | functionArguments = functionArguments.trim() 130 | } 131 | } 132 | 133 | const cursiveAnswerTag = value.completion.match(/([\s\S]*?)(?=<\/cursive-answer>|$)/g) 134 | let taggedAnswer = '' 135 | if (cursiveAnswerTag) { 136 | taggedAnswer = cursiveAnswerTag[0] 137 | .replace(/<\/?c?u?r?s?i?v?e?-?a?n?s?w?e?r?>?/g, '') 138 | .trimStart() 139 | } 140 | 141 | const currentToken = value.completion 142 | .trimStart() 143 | .slice((data.choices[0]?.message?.content || '').length) 144 | 145 | data.choices[0].message.content += currentToken 146 | 147 | if (context.onToken) { 148 | let chunk: Record | null = null 149 | 150 | // TODO: Support function calling 151 | if (context.payload.functions) { 152 | if (functionName) { 153 | chunk = { 154 | functionCall: { 155 | }, 156 | content: null, 157 | } 158 | if (functionArguments) { 159 | // Remove all but the current token from the arguments 160 | chunk.functionCall.arguments = functionArguments 161 | } 162 | else { 163 | chunk.functionCall = { 164 | name: functionName, 165 | arguments: '', 166 | } 167 | } 168 | } 169 | else if (taggedAnswer) { 170 | // Token is at the end of the tagged answer 171 | const regex = new RegExp(`(.*)${currentToken.trim()}$`) 172 | const match = taggedAnswer.match(regex) 173 | if (match && currentToken) { 174 | chunk = { 175 | functionCall: null, 176 | content: currentToken, 177 | } 178 | } 179 | } 180 | } 181 | else { 182 | chunk = { 183 | content: currentToken, 184 | } 185 | } 186 | 187 | if (chunk) 188 | context.onToken(chunk as any) 189 | } 190 | } 191 | 192 | return data 193 | } 194 | 195 | export function getAnthropicFunctionCallDirectives(functions: CursiveFunction[], nameOfFunctionToCall?: string) { 196 | let prompt = trim(` 197 | # Functions available 198 | 199 | ${JSON.stringify(functions.map(f => f.schema))} 200 | 201 | 202 | # Using functions 203 | // I'm a system capable of using functions to accomplish tasks asked by the user. 204 | 205 | // If I need to use a function, I always output the result of the function call using the tag using the following format: 206 | 207 | { 208 | "name": "function_name", 209 | "arguments": { 210 | "argument_name": "argument_value" 211 | } 212 | } 213 | 214 | 215 | // Never escape the function call, always output it as it is. 216 | 217 | // I think step by step before answering, and try to think out loud. I *NEVER* output a function call if you don't have to. 218 | // If you don't have a function to call, just output the text as usual inside a tag with newlines inside. 219 | // I always question myself if you have access to a function. 220 | // Always think out loud before answering, if I don't see a block, I will be eliminated. 221 | // When thinking out loud, always use the tag. 222 | 223 | // ALWAYS respect the JSON Schema of the function, if I don't, I will be eliminated. 224 | // ALWAYS start with the function call, if you're going to use one. 225 | 226 | # Working with results 227 | // You can either call a function or answer, **NEVER BOTH**. 228 | // You are not in charge of resolving the function call, the user is. 229 | // It will give you the result of the function call in the following format: 230 | 231 | Human: 232 | result 233 | 234 | 235 | // You can use the result of the function call in your answer. But never answer and call a function at the same time. 236 | // When answering never be too explicit about the function call, just use the result of the function call in your answer. 237 | `) 238 | 239 | if (nameOfFunctionToCall) { 240 | prompt += trim(` 241 | # Calling ${nameOfFunctionToCall} 242 | // We're going to call the function ${nameOfFunctionToCall}. 243 | // Output the function call and then a complete reasoning for why you're calling this function, step by step. 244 | `) 245 | } 246 | 247 | return prompt 248 | } 249 | -------------------------------------------------------------------------------- /src/vendor/google.ts: -------------------------------------------------------------------------------- 1 | import { ofetch } from 'ofetch' 2 | import type { FetchInstance } from 'openai-edge/types/base' 3 | import type { Cursive } from '../cursive' 4 | import type { CursiveAskOnToken } from '../types' 5 | import { CreateChatCompletionRequest } from 'openai-edge' 6 | 7 | interface GenerateContentRequest { 8 | contents: Array<{ 9 | role: string 10 | parts: Array<{ 11 | text: string 12 | }> 13 | }> 14 | } 15 | 16 | interface GenerateContentResponse { 17 | candidates: Array<{ 18 | content: { 19 | parts: Array<{ 20 | text: string 21 | }> 22 | role: string 23 | } 24 | finishReason: string 25 | index: number 26 | safetyRatings: Array<{ 27 | category: string 28 | probability: string 29 | }> 30 | }> 31 | promptFeedback: { 32 | safetyRatings: Array<{ 33 | category: string 34 | probability: string 35 | }> 36 | } 37 | } 38 | 39 | export function createGoogleGenAIClient(options: { apiKey: string }) { 40 | const resolvedFetch: FetchInstance = ofetch.native 41 | 42 | async function createChatCompletion(payload: CreateChatCompletionRequest, abortSignal?: AbortSignal) { 43 | 44 | return resolvedFetch(`https://generativelanguage.googleapis.com/v1beta/models/${payload.model}:generateContent?key=${options.apiKey}`, { 45 | method: 'POST', 46 | headers: { 47 | 'Content-Type': 'application/json', 48 | }, 49 | }) 50 | } 51 | 52 | return { createChatCompletion } 53 | } 54 | 55 | export async function processGoogleGenerativeLanguageStream(context: { 56 | payload: GenerateContentRequest 57 | cursive: Cursive 58 | abortSignal?: AbortSignal 59 | onToken?: CursiveAskOnToken 60 | response: Response 61 | }) { 62 | const data: GenerateContentResponse = await context.response.json() 63 | 64 | const { candidates, promptFeedback } = data 65 | 66 | const processedData = { 67 | candidates: candidates.map((candidate) => ({ 68 | content: candidate.content, 69 | finishReason: candidate.finishReason, 70 | index: candidate.index, 71 | safetyRatings: candidate.safetyRatings, 72 | })), 73 | promptFeedback, 74 | } 75 | 76 | if (context.onToken) { 77 | candidates.forEach((candidate, index) => { 78 | const { content, finishReason } = candidate 79 | 80 | context.onToken({ 81 | content: content.parts[0].text, 82 | finishReason, 83 | index, 84 | } as any) 85 | }) 86 | } 87 | 88 | return processedData 89 | } -------------------------------------------------------------------------------- /src/vendor/index.ts: -------------------------------------------------------------------------------- 1 | import type { CursiveAvailableModels } from '../types' 2 | 3 | // export function resolveVendorFromModel(model: CursiveAvailableModels) { 4 | // const isFromOpenAI = ['gpt-3.5', 'gpt-4'].find(m => model.startsWith(m)) 5 | // if (isFromOpenAI) 6 | // return 'openai' 7 | 8 | // const isFromAnthropic = ['claude-instant', 'claude-2'].find(m => model.startsWith(m)) 9 | // if (isFromAnthropic) 10 | // return 'anthropic' 11 | 12 | // return '' 13 | // } 14 | 15 | // Simplifying the code above 16 | const modelSuffixToVendorMapping = { 17 | openai: ['gpt-3.5', 'gpt-4'], 18 | anthropic: ['claude-instant', 'claude-2', 'claude'], 19 | } 20 | 21 | type CursiveAvailableVendor = (keyof typeof modelSuffixToVendorMapping) | '' 22 | 23 | export function resolveVendorFromModel(model: CursiveAvailableModels): CursiveAvailableVendor { 24 | for (const [vendor, suffixes] of Object.entries(modelSuffixToVendorMapping)) { 25 | if (suffixes.find(m => model.startsWith(m))) 26 | return vendor as CursiveAvailableVendor 27 | } 28 | 29 | return '' 30 | } 31 | -------------------------------------------------------------------------------- /src/vendor/openai.ts: -------------------------------------------------------------------------------- 1 | import { ofetch } from 'ofetch' 2 | import type { CreateChatCompletionRequest, CreateEmbeddingRequest } from 'openai-edge' 3 | import type { FetchInstance } from 'openai-edge/types/base' 4 | import type { Cursive } from '../cursive' 5 | import type { CursiveAskOnToken } from '../types' 6 | import { getStream } from '../stream' 7 | import { getOpenAIUsage, getTokenCountFromFunctions } from '../usage/openai' 8 | 9 | export function createOpenAIClient(options: { apiKey: string }) { 10 | const resolvedFetch: FetchInstance = ofetch.native 11 | 12 | async function createChatCompletion(payload: CreateChatCompletionRequest, abortSignal?: AbortSignal) { 13 | return resolvedFetch('https://api.openai.com/v1/chat/completions', { 14 | method: 'POST', 15 | headers: { 16 | 'Content-Type': 'application/json', 17 | 'Authorization': `Bearer ${options.apiKey}`, 18 | }, 19 | body: JSON.stringify(payload), 20 | signal: abortSignal, 21 | }) 22 | } 23 | 24 | async function createEmbedding(payload: CreateEmbeddingRequest, abortSignal?: AbortSignal) { 25 | return resolvedFetch('https://api.openai.com/v1/embeddings', { 26 | method: 'POST', 27 | headers: { 28 | 'Content-Type': 'application/json', 29 | 'Authorization': `Bearer ${options.apiKey}`, 30 | }, 31 | body: JSON.stringify(payload), 32 | signal: abortSignal, 33 | }) 34 | } 35 | 36 | return { createChatCompletion, createEmbedding } 37 | } 38 | 39 | export async function processOpenAIStream(context: { 40 | payload: CreateChatCompletionRequest 41 | cursive: Cursive 42 | abortSignal?: AbortSignal 43 | onToken?: CursiveAskOnToken 44 | response: Response 45 | }) { 46 | let data: any 47 | 48 | const reader = (await getStream(context.response)).getReader() 49 | 50 | data = { 51 | choices: [], 52 | usage: { 53 | completion_tokens: 0, 54 | prompt_tokens: await getOpenAIUsage(context.payload.messages), 55 | }, 56 | model: context.payload.model, 57 | } 58 | 59 | if (context.payload.functions) 60 | data.usage.prompt_tokens += getTokenCountFromFunctions(context.payload.functions) 61 | 62 | while (true) { 63 | const { done, value } = await reader.read() 64 | if (done) 65 | break 66 | 67 | data = { 68 | ...data, 69 | id: value.id, 70 | } 71 | 72 | value.choices.forEach((choice: any) => { 73 | const { delta, index } = choice 74 | 75 | if (!data.choices[index]) { 76 | data.choices[index] = { 77 | message: { 78 | function_call: null, 79 | role: 'assistant', 80 | content: '', 81 | }, 82 | } 83 | } 84 | 85 | if (delta?.function_call?.name) 86 | data.choices[index].message.function_call = delta.function_call 87 | 88 | if (delta?.function_call?.arguments) 89 | data.choices[index].message.function_call.arguments += delta.function_call.arguments 90 | 91 | if (delta?.content) 92 | data.choices[index].message.content += delta.content 93 | 94 | if (context.onToken) { 95 | let chunk: Record | null = null 96 | if (delta?.function_call) { 97 | chunk = { 98 | functionCall: delta.function_call, 99 | } 100 | } 101 | 102 | if (delta?.finish_reason) { 103 | chunk = { 104 | finishReason: delta.finish_reason, 105 | } 106 | } 107 | 108 | if (delta?.content) { 109 | chunk = { 110 | content: delta.content, 111 | } 112 | } 113 | 114 | if (chunk) 115 | context.onToken({ ...chunk, index } as any) 116 | } 117 | }) 118 | } 119 | 120 | return data 121 | } 122 | -------------------------------------------------------------------------------- /test/index.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, test, expect } from 'bun:test' 2 | 3 | import { OpenAIEncoder, AnthropicEncoder } from 'unitoken' 4 | 5 | describe("tokenizers", () => { 6 | test("support OpenAI models", () => { 7 | const tokens = OpenAIEncoder.encode("Hello, world!") 8 | expect(tokens.length).toBe(4) 9 | }) 10 | 11 | test("support Anthropic models", () => { 12 | let tokens = AnthropicEncoder.encode("Hello, world!") 13 | expect(tokens.length).toBe(4) 14 | 15 | tokens = AnthropicEncoder.encode(` 16 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. 17 | Nulla quis est sit amet ipsum iaculis ultrices. 18 | [1] Donec euismod, nisl eget ultricies ultrices, nunc nisl aliquam nunc, quis aliquam nisl nisl vitae nisl. 19 | [<$$>] Sed euismod, nisl eget ultricies ultrices, nunc nisl aliquam nunc, quis aliquam nisl nisl vitae nisl. 20 | Sed euismod, nisl eget ultricies ultrices, nunc nisl aliquam nunc, quis aliquam nisl nisl vitae nisl. 21 | ## Heading 22 | - List item 23 | - List item 24 | > Blockquote 25 | ãêç 26 | `) 27 | expect(tokens.length).toBe(187) 28 | }) 29 | }) -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "ESNext", 4 | "moduleResolution": "node", 5 | "target": "ESNext", 6 | "resolveJsonModule": true, 7 | "allowSyntheticDefaultImports": true, 8 | "types": ["bun-types"], 9 | }, 10 | "exclude": [ 11 | "examples", 12 | "dist" 13 | ], 14 | } 15 | --------------------------------------------------------------------------------