├── .npmrc ├── tests ├── unit │ ├── .gitignore │ ├── schema.test.ts │ ├── ai-providers.test.ts │ └── generator.test.ts ├── types │ ├── README.md │ ├── schema.test-d.ts │ └── fastify.test-d.ts ├── utils │ ├── mocks │ │ ├── index.ts │ │ ├── base.ts │ │ ├── llama2.ts │ │ ├── ollama.ts │ │ ├── mistral.ts │ │ ├── open-ai.ts │ │ └── azure.ts │ ├── auth.ts │ └── stackable.ts └── e2e │ ├── auth.test.ts │ ├── rate-limiting.test.ts │ └── api.test.ts ├── README.md ├── .npmignore ├── static ├── images │ ├── icons │ │ ├── arrow-left.svg │ │ ├── arrow-right.svg │ │ ├── arrow-long-right.svg │ │ ├── checkmark.svg │ │ ├── error.svg │ │ ├── regenerate.svg │ │ ├── edit.svg │ │ └── copy.svg │ ├── avatars │ │ ├── you.svg │ │ └── platformatic.svg │ ├── platformatic-logo.svg │ └── favicon.svg ├── styles │ ├── common.css │ ├── index.css │ └── chat.css ├── chat.html ├── index.html └── scripts │ └── chat.js ├── cli ├── start.ts └── create.ts ├── lib ├── templates │ └── types.ts ├── generator.ts └── schema.ts ├── renovate.json ├── .gitignore ├── NOTICE ├── ai-providers ├── provider.ts ├── event.ts ├── ollama.ts ├── mistral.ts ├── azure.ts ├── open-ai.ts └── llama2.ts ├── tsconfig.json ├── plugins ├── auth.ts ├── api.ts ├── warp.ts └── rate-limiting.ts ├── .github └── workflows │ ├── lint-md.yml │ └── ci.yml ├── docs ├── config.md ├── auth.md ├── rest-api.md ├── rate-limiting.md ├── add-ai-provider.md └── plugin-api.md ├── package.json ├── index.ts ├── CONTRIBUTING.md ├── index.d.ts ├── LICENSE └── config.d.ts /.npmrc: -------------------------------------------------------------------------------- 1 | package-lock=true 2 | -------------------------------------------------------------------------------- /tests/unit/.gitignore: -------------------------------------------------------------------------------- 1 | tmp/ 2 | -------------------------------------------------------------------------------- /tests/types/README.md: -------------------------------------------------------------------------------- 1 | # Type Tests 2 | Tests for public-facing types 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This repository is archived in favor of https://github.com/platformatic/ai-warp 2 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | tests 2 | /static 3 | /lib 4 | /plugins 5 | /cli 6 | CONTIBUTING.md 7 | renovate.json 8 | tsconfig.json 9 | .github 10 | ai-warp-app* 11 | *gguf 12 | /index.* 13 | dist/tsconfig.tsbuildinfo 14 | /ai-providers 15 | -------------------------------------------------------------------------------- /static/images/icons/arrow-left.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /static/images/icons/arrow-right.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /static/images/icons/arrow-long-right.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /cli/start.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import stackable from '../index.js' 3 | import { start } from '@platformatic/service' 4 | import { printAndExitLoadConfigError } from '@platformatic/config' 5 | 6 | start(stackable, process.argv.splice(2)).catch(printAndExitLoadConfigError) 7 | -------------------------------------------------------------------------------- /static/images/icons/checkmark.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /static/images/icons/error.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /static/images/avatars/you.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /lib/templates/types.ts: -------------------------------------------------------------------------------- 1 | export function generateGlobalTypesFile (npmPackageName: string): string { 2 | return `import { FastifyInstance } from 'fastify' 3 | import { AiWarpConfig, PlatformaticApp } from '${npmPackageName}' 4 | 5 | declare module 'fastify' { 6 | interface FastifyInstance { 7 | platformatic: PlatformaticApp 8 | } 9 | } 10 | ` 11 | } 12 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base" 5 | ], 6 | "packageRules": [ 7 | { 8 | "groupName": "Safe automerge", 9 | "matchUpdateTypes": ["minor", "patch", "pin", "digest"], 10 | "automerge": true 11 | } 12 | ], 13 | "lockFileMaintenance": { 14 | "enabled": true, 15 | "automerge": true 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | .DS_Store 3 | 4 | # dotenv environment variable files 5 | .env 6 | 7 | # database files 8 | *.sqlite 9 | *.sqlite3 10 | 11 | # Logs 12 | logs 13 | *.log 14 | npm-debug.log* 15 | yarn-debug.log* 16 | yarn-error.log* 17 | lerna-debug.log* 18 | .pnpm-debug.log* 19 | 20 | # Dependency directories 21 | node_modules/ 22 | 23 | # ctags 24 | tags 25 | 26 | # clinicjs 27 | .clinic/ 28 | 29 | ai-warp-app*/ 30 | 31 | *.gguf 32 | -------------------------------------------------------------------------------- /tests/utils/mocks/index.ts: -------------------------------------------------------------------------------- 1 | import { after } from 'node:test' 2 | import { mockAzure } from './azure.js' 3 | import { mockMistralApi } from './mistral.js' 4 | import { mockOllama } from './ollama.js' 5 | import { mockOpenAiApi } from './open-ai.js' 6 | 7 | export function mockAllProviders (): void { 8 | mockOpenAiApi() 9 | mockMistralApi() 10 | mockOllama() 11 | 12 | const azureMock = mockAzure() 13 | after(() => { 14 | azureMock.close() 15 | }) 16 | } 17 | -------------------------------------------------------------------------------- /tests/utils/auth.ts: -------------------------------------------------------------------------------- 1 | import { createSigner } from 'fast-jwt' 2 | import { AiWarpConfig } from '../../config.js' 3 | 4 | export const authConfig: AiWarpConfig['auth'] = { 5 | required: true, 6 | jwt: { 7 | secret: 'secret' 8 | } 9 | } 10 | 11 | export function createToken (payload: string | Buffer | { [key: string]: any }, opts = {}): string { 12 | const signSync = createSigner({ 13 | key: 'secret', 14 | expiresIn: '1h', 15 | ...opts 16 | }) 17 | 18 | return signSync(payload) 19 | } 20 | -------------------------------------------------------------------------------- /static/images/icons/regenerate.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /tests/unit/schema.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-floating-promises */ 2 | import { it } from 'node:test' 3 | import assert from 'node:assert' 4 | import { join } from 'node:path' 5 | import { readFile } from 'node:fs/promises' 6 | import { schema } from '../../lib/schema.js' 7 | 8 | it('updates schema version correctly', async () => { 9 | const pkgJsonText = await readFile(join(import.meta.dirname, '..', '..', 'package.json'), 'utf8') 10 | const pkgJson = JSON.parse(pkgJsonText) 11 | assert.strictEqual(schema.version, pkgJson.version) 12 | }) 13 | -------------------------------------------------------------------------------- /static/images/icons/edit.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Platformatic 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /ai-providers/provider.ts: -------------------------------------------------------------------------------- 1 | import { ReadableStream } from 'node:stream/web' 2 | import createError from '@fastify/error' 3 | 4 | export type ChatHistory = Array<{ prompt: string, response: string }> 5 | 6 | export type StreamChunkCallback = (response: string) => Promise 7 | 8 | export interface AiProvider { 9 | ask: (prompt: string, chatHistory?: ChatHistory) => Promise 10 | askStream: (prompt: string, chunkCallback?: StreamChunkCallback, chatHistory?: ChatHistory) => Promise 11 | } 12 | 13 | export const NoContentError = createError<[string]>('NO_CONTENT', '%s didn\'t return any content') 14 | -------------------------------------------------------------------------------- /static/images/icons/copy.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "NodeNext", 4 | "esModuleInterop": true, 5 | "moduleResolution": "NodeNext", 6 | "target": "ESNext", 7 | "sourceMap": true, 8 | "pretty": true, 9 | "noEmitOnError": true, 10 | "incremental": true, 11 | "strict": true, 12 | "outDir": "dist", 13 | "declaration": true, 14 | "skipLibCheck": true 15 | }, 16 | "watchOptions": { 17 | "watchFile": "fixedPollingInterval", 18 | "watchDirectory": "fixedPollingInterval", 19 | "fallbackPolling": "dynamicPriority", 20 | "synchronousWatchDirectory": true, 21 | "excludeDirectories": [ 22 | "**/node_modules", 23 | "dist" 24 | ] 25 | }, 26 | "exclude": [ 27 | "ai-warp-app", 28 | "dist" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /plugins/auth.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line 2 | /// 3 | import { FastifyInstance } from 'fastify' 4 | import createError from '@fastify/error' 5 | import fastifyPlugin from 'fastify-plugin' 6 | 7 | const UnauthorizedError = createError('UNAUTHORIZED', 'Unauthorized', 401) 8 | 9 | export default fastifyPlugin(async (fastify: FastifyInstance) => { 10 | const { config } = fastify.platformatic 11 | 12 | fastify.addHook('onRequest', async (request) => { 13 | await request.extractUser() 14 | 15 | const isAuthRequired = config.auth?.required !== undefined && config.auth?.required 16 | const isMissingUser = request.user === undefined || request.user === null 17 | if (isAuthRequired && isMissingUser) { 18 | throw new UnauthorizedError() 19 | } 20 | }) 21 | }) 22 | -------------------------------------------------------------------------------- /tests/utils/mocks/base.ts: -------------------------------------------------------------------------------- 1 | import { MockAgent, setGlobalDispatcher } from 'undici' 2 | 3 | export const MOCK_CONTENT_RESPONSE = 'asd123' 4 | 5 | export const MOCK_STREAMING_CONTENT_CHUNKS = [ 6 | 'chunk1', 7 | 'chunk2', 8 | 'chunk3' 9 | ] 10 | 11 | /** 12 | * @returns The full body that should be returned from the stream endpoint 13 | */ 14 | export function buildExpectedStreamBodyString (): string { 15 | let body = '' 16 | for (const chunk of MOCK_STREAMING_CONTENT_CHUNKS) { 17 | body += `event: content\ndata: {"response":"${chunk}"}\n\n` 18 | } 19 | return body 20 | } 21 | 22 | export const MOCK_AGENT = new MockAgent() 23 | 24 | let isMockAgentEstablished = false 25 | export function establishMockAgent (): void { 26 | if (isMockAgentEstablished) { 27 | return 28 | } 29 | setGlobalDispatcher(MOCK_AGENT) 30 | isMockAgentEstablished = true 31 | } 32 | -------------------------------------------------------------------------------- /tests/utils/stackable.ts: -------------------------------------------------------------------------------- 1 | import { buildServer } from '@platformatic/service' 2 | import { FastifyInstance } from 'fastify' 3 | import stackable from '../../index.js' 4 | import { AiWarpConfig } from '../../config.js' 5 | 6 | declare module 'fastify' { 7 | interface FastifyInstance { 8 | start: () => Promise 9 | } 10 | } 11 | 12 | let apps = 0 13 | export function getPort (): number { 14 | apps++ 15 | return 3042 + apps 16 | } 17 | 18 | export async function buildAiWarpApp (config: AiWarpConfig): Promise<[FastifyInstance, number]> { 19 | const port = getPort() 20 | const app = await buildServer({ 21 | server: { 22 | port, 23 | forceCloseConnections: true, 24 | healthCheck: false, 25 | logger: { 26 | level: 'silent' 27 | } 28 | }, 29 | service: { 30 | openapi: true 31 | }, 32 | ...config 33 | }, stackable) 34 | 35 | return [app, port] 36 | } 37 | -------------------------------------------------------------------------------- /ai-providers/event.ts: -------------------------------------------------------------------------------- 1 | import { FastifyError } from 'fastify' 2 | import fastJson from 'fast-json-stringify' 3 | 4 | const stringifyEventData = fastJson({ 5 | title: 'Stream Event Data', 6 | type: 'object', 7 | properties: { 8 | // Success 9 | response: { type: 'string' }, 10 | // Error 11 | code: { type: 'string' }, 12 | message: { type: 'string' } 13 | } 14 | }) 15 | 16 | export interface AiStreamEventContent { 17 | response: string 18 | } 19 | 20 | export type AiStreamEvent = { 21 | event: 'content' 22 | data: AiStreamEventContent 23 | } | { 24 | event: 'error' 25 | data: FastifyError 26 | } 27 | 28 | /** 29 | * Encode an event to the Event Stream format 30 | * @see https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#event_stream_format 31 | */ 32 | export function encodeEvent ({ event, data }: AiStreamEvent): Uint8Array { 33 | const jsonString = stringifyEventData(data) 34 | const eventString = `event: ${event}\ndata: ${jsonString}\n\n` 35 | 36 | return new TextEncoder().encode(eventString) 37 | } 38 | -------------------------------------------------------------------------------- /static/styles/common.css: -------------------------------------------------------------------------------- 1 | @import url('https://fonts.googleapis.com/css2?family=Inter:wght@100..900&display=swap'); 2 | 3 | body { 4 | background-color: #00050B; 5 | padding: 0; 6 | margin: 0; 7 | color: white; 8 | font-family: "Inter", sans-serif; 9 | font-optical-sizing: auto; 10 | font-weight: 100; 11 | font-style: normal; 12 | font-variation-settings: "slnt" 0; 13 | } 14 | 15 | #navbar { 16 | padding-top: 5px; 17 | padding-bottom: 5px; 18 | width: 100%; 19 | border-bottom: 2px solid #FFFFFF26; 20 | } 21 | 22 | #navbar-logo { 23 | padding-left: 60px; 24 | } 25 | 26 | #bottom-links { 27 | margin-top: 30px; 28 | /* 30px on the bottom so there's a little space between it and the end of the page */ 29 | margin-bottom: 30px; 30 | text-align: center; 31 | } 32 | 33 | #bottom-links a { 34 | padding-top: 12px; 35 | padding-bottom: 12px; 36 | padding-left: 8px; 37 | padding-right: 8px; 38 | margin-right: 8px; 39 | border: 1px solid #FFFFFFB2; 40 | border-radius: 4px; 41 | text-decoration: none; 42 | color: #FFFFFF; 43 | } 44 | -------------------------------------------------------------------------------- /.github/workflows/lint-md.yml: -------------------------------------------------------------------------------- 1 | name: Lint Markdown 2 | 3 | on: 4 | push: 5 | branches-ignore: 6 | - main 7 | paths: 8 | - "**/*.md" 9 | pull_request: 10 | paths: 11 | - "**/*.md" 12 | 13 | jobs: 14 | setup-node-modules: 15 | runs-on: ubuntu-latest 16 | timeout-minutes: 15 17 | steps: 18 | - name: Git Checkout 19 | uses: actions/checkout@09d2acae674a48949e3602304ab46fd20ae0c42f 20 | 21 | - name: Setup Node 22 | uses: actions/setup-node@v4 23 | with: 24 | node-version: 20 25 | cache: 'npm' 26 | 27 | - name: Install Dependencies 28 | run: npm install 29 | 30 | lint-md: 31 | name: Linting Markdown 32 | runs-on: ubuntu-latest 33 | needs: setup-node-modules 34 | steps: 35 | - name: Git Checkout 36 | uses: actions/checkout@09d2acae674a48949e3602304ab46fd20ae0c42f 37 | 38 | - name: Setup Node 39 | uses: actions/setup-node@v4 40 | with: 41 | node-version: 20 42 | cache: 'npm' 43 | 44 | - name: Install Dependencies 45 | run: npm install 46 | 47 | - name: Run Markdown Linting 48 | run: npm run lint-md 49 | -------------------------------------------------------------------------------- /static/chat.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Chat - AI Warp 7 | 8 | 9 | 10 | 11 | 12 | 13 | 18 | 19 |
20 | 21 |
22 | 23 | 26 |
27 | 28 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /docs/config.md: -------------------------------------------------------------------------------- 1 | # AI Warp Configuration 2 | 3 | Documentation on AI Warp's configuration options. 4 | 5 | ## How to Configure AI Warp 6 | 7 | AI Warp can be configured in your applications's Platformatic config. 8 | 9 | ## Config Options 10 | 11 | ### `aiProvider` 12 | 13 | * `object` 14 | 15 | > \[!NOTE]\ 16 | > This is a required configuration option. 17 | 18 | This configuration option tells AI Warp what AI provider to use. 19 | 20 | ```json 21 | { 22 | "aiProvider": { 23 | "openai": { 24 | "model": "gpt-3.5-turbo", 25 | "apiKey": "{PLT_OPENAI_API_KEY}" // reads from environment variables 26 | } 27 | } 28 | } 29 | ``` 30 | 31 | ### `promptDecorators` 32 | 33 | * `object` 34 | 35 | Tells AI Warp to append a prefix and/or a suffix to a prompt. 36 | 37 |
38 | Example usage 39 | 40 | ```json 41 | { 42 | "promptDecorators": { 43 | "prefix": "Hello AI! Your prompt is as follows: \n", 44 | "suffix": "\nThank you!" 45 | } 46 | } 47 | ``` 48 |
49 | 50 | ### `auth` 51 | 52 | * `object` 53 | 54 | Configure authentication for AI Warp. See [auth.md](./auth.md) for more information. 55 | 56 | ### `rateLimiting` 57 | 58 | * `object` 59 | 60 | Configure rate limiting for AI Warp. See [rate-limiting.md](./rate-limiting.md) for more information. 61 | -------------------------------------------------------------------------------- /cli/create.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import { join } from 'node:path' 3 | import { parseArgs } from 'node:util' 4 | import { Generator } from '../lib/generator.js' 5 | 6 | async function execute (): Promise { 7 | const args = parseArgs({ 8 | args: process.argv.slice(2), 9 | options: { 10 | dir: { 11 | type: 'string', 12 | default: join(process.cwd(), 'ai-warp-app') 13 | }, 14 | port: { type: 'string', default: '3042' }, 15 | hostname: { type: 'string', default: '0.0.0.0' }, 16 | plugin: { type: 'boolean' }, 17 | tests: { type: 'boolean' }, 18 | typescript: { type: 'boolean' }, 19 | git: { type: 'boolean' }, 20 | localSchema: { type: 'boolean' } 21 | } 22 | }) 23 | 24 | const generator = new Generator() 25 | 26 | const config = { 27 | port: parseInt(args.values.port), 28 | hostname: args.values.hostname, 29 | plugin: args.values.plugin, 30 | tests: args.values.tests, 31 | typescript: args.values.typescript, 32 | initGitRepository: args.values.git, 33 | targetDirectory: args.values.dir 34 | } 35 | 36 | generator.setConfig(config) 37 | 38 | await generator.run() 39 | 40 | console.log('Application created successfully! Run `npm run start` to start an application.') 41 | } 42 | 43 | execute().catch(err => { 44 | throw err 45 | }) 46 | -------------------------------------------------------------------------------- /docs/auth.md: -------------------------------------------------------------------------------- 1 | # Authentication 2 | 3 | Documentation on how to configure and use AI Warp's authentication. 4 | 5 | ## Configuring 6 | 7 | Configuring authentication can be done via your Platformatic config file under the `auth` object. E.g. 8 | 9 | ```json 10 | // platformatic.json 11 | { 12 | "auth": { 13 | // ... 14 | } 15 | } 16 | ``` 17 | 18 | We utilize [fastify-user](https://github.com/platformatic/fastify-user) to do authentication, so you 19 | can pass in any configuration options for it in the `auth` object. 20 | 21 | AI Warp-specific options: 22 | 23 | * `required` (`boolean`) - If true, any unauthenticated users will receive a 401 status code and body. 24 | 25 | ### Example 26 | 27 | This makes authentication required and accepts JWTs signed with the secret `abc123`: 28 | 29 | ```json 30 | { 31 | "auth": { 32 | "required": true, 33 | "jwt": { 34 | "secret": "abc123" 35 | } 36 | } 37 | } 38 | ``` 39 | 40 | ## Using 41 | 42 | By default, [fastify-user](https://github.com/platformatic/fastify-user) reads the `Authorization` header. 43 | 44 | You can configure AI Warp to read the token from cookies in your Platformatic config: 45 | 46 | ```json 47 | { 48 | "auth": { 49 | "jwt": { 50 | "cookie": { 51 | "cookieName": "token", 52 | "signed": false 53 | } 54 | } 55 | } 56 | } 57 | ``` 58 | -------------------------------------------------------------------------------- /tests/utils/mocks/llama2.ts: -------------------------------------------------------------------------------- 1 | import esmock from 'esmock' 2 | import { MOCK_CONTENT_RESPONSE, MOCK_STREAMING_CONTENT_CHUNKS } from './base.js' 3 | 4 | // eslint-disable-next-line @typescript-eslint/no-extraneous-class 5 | class LlamaModelMock {} 6 | 7 | class LlamaContextMock { 8 | decode (token: number[]): string { 9 | const chunkIndex = token[0] 10 | return MOCK_STREAMING_CONTENT_CHUNKS[chunkIndex] 11 | } 12 | } 13 | 14 | interface PromptOptions { 15 | onToken: (token: number[]) => void 16 | } 17 | 18 | class LlamaChatSessionMock { 19 | context: LlamaContextMock 20 | 21 | constructor ({ context }: { context: LlamaContextMock }) { 22 | this.context = context 23 | } 24 | 25 | async prompt (_: string, options?: PromptOptions): Promise { 26 | if (options !== undefined) { 27 | for (let i = 0; i < MOCK_STREAMING_CONTENT_CHUNKS.length; i++) { 28 | // Send an array with just one element that's the chunk number 29 | options.onToken([i]) 30 | } 31 | } 32 | 33 | return MOCK_CONTENT_RESPONSE 34 | } 35 | } 36 | 37 | export async function mockLlama2 (): Promise { 38 | const llama2Provider = await esmock('../../../ai-providers/llama2.ts', { 39 | 'node-llama-cpp': { 40 | LlamaModel: LlamaModelMock, 41 | LlamaContext: LlamaContextMock, 42 | LlamaChatSession: LlamaChatSessionMock 43 | } 44 | }) 45 | 46 | return llama2Provider 47 | } 48 | -------------------------------------------------------------------------------- /tests/e2e/auth.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-floating-promises */ 2 | import { it } from 'node:test' 3 | import assert from 'node:assert' 4 | import { buildAiWarpApp } from '../utils/stackable.js' 5 | import { AiWarpConfig } from '../../config.js' 6 | import { authConfig, createToken } from '../utils/auth.js' 7 | import { mockAllProviders } from '../utils/mocks/index.js' 8 | mockAllProviders() 9 | 10 | const aiProvider: AiWarpConfig['aiProvider'] = { 11 | openai: { 12 | model: 'gpt-3.5-turbo', 13 | apiKey: '' 14 | } 15 | } 16 | 17 | it('returns 401 for unauthorized user', async () => { 18 | const [app, port] = await buildAiWarpApp({ 19 | aiProvider, 20 | auth: { 21 | required: true 22 | } 23 | }) 24 | 25 | try { 26 | await app.start() 27 | 28 | const response = await fetch(`http://localhost:${port}`) 29 | assert.strictEqual(response.status, 401) 30 | } finally { 31 | await app.close() 32 | } 33 | }) 34 | 35 | it('returns 200 for authorized user', async () => { 36 | const [app, port] = await buildAiWarpApp({ 37 | aiProvider, 38 | auth: authConfig 39 | }) 40 | 41 | try { 42 | await app.start() 43 | 44 | const response = await fetch(`http://localhost:${port}`, { 45 | headers: { 46 | Authorization: `Bearer ${createToken({ asd: '123' })}` 47 | } 48 | }) 49 | assert.strictEqual(response.status, 200) 50 | } finally { 51 | await app.close() 52 | } 53 | }) 54 | -------------------------------------------------------------------------------- /docs/rest-api.md: -------------------------------------------------------------------------------- 1 | # REST API Endpoints 2 | 3 | Documentation on AI Warp's REST API. 4 | 5 | For information on authentication, see [here](./auth.md). 6 | 7 | For information on rate limiting, see [here](./rate-limiting.md) 8 | 9 | ## Endpoints 10 | 11 | ### POST `/api/v1/prompt` 12 | 13 | Prompt the AI Provider and receive the full response. 14 | 15 |
16 | Body 17 | 18 | ```json 19 | { "prompt": "What's 1+1?" } 20 | ``` 21 |
22 | 23 |
24 | Response 25 | 26 | ```json 27 | { "response": "..." } 28 | ``` 29 |
30 | 31 | ### POST `/api/v1/stream` 32 | 33 | Prompt the AI Provider and receive a streamed response. This endpoint supports [Server Side Events](https://html.spec.whatwg.org/multipage/server-sent-events.html). 34 | 35 | Event types: 36 | 37 | * `content` - Response chunk 38 | * `error` - An error has occured and the stream is closed. 39 | 40 |
41 | Body 42 | 43 | ```json 44 | { "prompt": "What's 1+1?" } 45 | ``` 46 |
47 | 48 |
49 | Success response 50 | 51 | ``` 52 | event: content 53 | data: {"response": "..."} 54 | 55 | event: content 56 | data: {"response": "..."} 57 | ``` 58 |
59 | 60 |
61 | Error response 62 | 63 | ``` 64 | event: error 65 | data: {"code":"...","message":"..."} 66 | ``` 67 |
68 | 69 | When there is no more chunks to return or an error occurs, the stream is closed. 70 | -------------------------------------------------------------------------------- /static/images/avatars/platformatic.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /docs/rate-limiting.md: -------------------------------------------------------------------------------- 1 | # Rate Limiting 2 | 3 | Documentation on configuring AI Warp's rate limiting. 4 | 5 | ## Configuring 6 | 7 | Configuring rate limiting can be done via your Platformatic config file under the `rateLimiting` object. E.g. 8 | 9 | ```json 10 | // platformatic.json 11 | { 12 | "rateLimiting": { 13 | // ... 14 | } 15 | } 16 | ``` 17 | 18 | We utilize the [@fastify/rate-limit](https://github.com/fastify/fastify-rate-limit) module for rate limiting. You can 19 | pass in any configuration options from it into the `rateLimiting` object. 20 | 21 | For defining the callbacks allowed by that module, set them in the `fastify.ai.rateLimiting` object. 22 | See the [Plugin API docs](./plugin-api.md#fastifyairatelimitingmax) for more information. 23 | 24 | ## Determining a client's request limit from JWT claims 25 | 26 | AI Warp provides an easy and simple way to decide a client's request limit based off of JWT claims. 27 | This is useful for say differentiating between free and premium users, where premium users get a higher 28 | request limit. 29 | 30 | > \[!NOTE]\ 31 | > This requires authentication to be enabled. Documentation for configuring authentication is available [here](./auth.md). 32 | 33 | You can configure this within your Platformatic config under the `rateLimiting.maxByClaims` array: 34 | 35 | ```json 36 | { 37 | "rateLimiting": { 38 | "maxByClaims": [ 39 | { 40 | "claim": "name-of-the-claim", 41 | "claimValue": "value-necessary", 42 | "max": 10 43 | } 44 | ] 45 | } 46 | } 47 | ``` 48 | 49 | So, for differentiating between free and premium users, you could do: 50 | 51 | ```json 52 | { 53 | "rateLimiting": { 54 | "max": 100, // request limit for free users 55 | "maxByClaims": { 56 | { 57 | "claim": "userType", 58 | "claimValue": "premium", 59 | "max": 1000 60 | } 61 | } 62 | } 63 | } 64 | ``` 65 | -------------------------------------------------------------------------------- /docs/add-ai-provider.md: -------------------------------------------------------------------------------- 1 | # Adding a new AI Provider 2 | Documentation on how to support a new AI provider. 3 | 4 | ## Steps 5 | 6 | ### 1. Setup your developer environment 7 | 8 | See [CONTRIBUTING](../CONTRIBUTING.md). 9 | 10 | ### 2. Add it to the Config Schema 11 | 12 | The `aiProvider` property in the config schema ([lib/schema.ts](../lib/schema.ts)) needs to be updated to allow for inputting any necessary information for this AI provider (e.g. model name, api key). Don't forget to rebuild the config! 13 | 14 | ### 3. Creating the Provider class 15 | 16 | Implement the `Provider` interface ([ai-providers/provider.ts](../ai-providers/provider.ts)) in a file also under [ai-providers/](../ai-providers/) (e.g. [ai-providers/open-ai.ts](../ai-providers/open-ai.ts)). 17 | 18 | Ensure that the `askStream` response returns a Node.js-builtin `ReadableStream` that outputs the expected format defined in the [REST API docs](./rest-api.md). 19 | 20 | ### 4. Add the provider to the `build` function in the `warp` plugin 21 | 22 | See [plugins/warp.ts](https://github.com/platformatic/ai-warp/blob/b9cddeedf8609d1c2ce3efcfdd84a739150a1e91/plugins/warp.ts#L12) `build()`. 23 | 24 | ### 5. Add the provider to the generator code 25 | 26 | See [lib/generator.ts](https://github.com/platformatic/ai-warp/blob/b9cddeedf8609d1c2ce3efcfdd84a739150a1e91/lib/generator.ts#L64-L88). 27 | 28 | ### 6. Unit Tests 29 | 30 | Add provider to [tests/unit/ai-providers.test.ts](https://github.com/platformatic/ai-warp/blob/b9cddeedf8609d1c2ce3efcfdd84a739150a1e91/tests/unit/ai-providers.test.ts#L11). 31 | 32 | ### 7. E2E Tests 33 | 34 | Add provider config to [tests/e2e/api.test.ts](https://github.com/platformatic/ai-warp/blob/b9cddeedf8609d1c2ce3efcfdd84a739150a1e91/tests/e2e/api.test.ts#L17-L36). 35 | 36 | ### 8. Type Tests 37 | 38 | Add the provider config to the schema tests [tests/types/schema.test-d.ts](https://github.com/platformatic/ai-warp/blob/main/tests/types/schema.test-d.ts). 39 | -------------------------------------------------------------------------------- /static/images/platformatic-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /static/styles/index.css: -------------------------------------------------------------------------------- 1 | #main-illustration { 2 | margin-top: 16px; 3 | width: 100%; 4 | text-align: center; 5 | position: absolute; 6 | } 7 | 8 | #main-illustration img { 9 | width: 957px; 10 | } 11 | 12 | #greeting { 13 | padding-top: 270px; 14 | text-align: center; 15 | line-height: 2.7rem; 16 | } 17 | 18 | #greeting-top-text { 19 | color: rgba(255, 255, 255, 0.7); 20 | } 21 | 22 | #prompt-suggestions { 23 | width: 50%; 24 | margin-left: 25%; 25 | display: flex; 26 | } 27 | 28 | .prompt-suggestion-column { 29 | width: 100%; 30 | margin-right: 8px; 31 | display: flex; 32 | flex-direction: column; 33 | align-content: right; 34 | } 35 | 36 | .prompt-suggestion { 37 | width: 100%; 38 | margin-top: 8px; 39 | border: 1px solid #FFFFFF4D; 40 | border-radius: 4px; 41 | } 42 | 43 | .prompt-suggestion p { 44 | margin-left: 8px; 45 | margin-right: 8px; 46 | } 47 | 48 | .prompt-suggestion-title { 49 | margin-top: 8px; 50 | font-weight: 600; 51 | } 52 | 53 | .prompt-suggestion-subtitle { 54 | color: #FFFFFFB2; 55 | margin-bottom: 8px; 56 | } 57 | 58 | .prompt-suggestion-subtitle a { 59 | float: right; 60 | } 61 | 62 | #custom-prompt { 63 | width: 50%; 64 | margin-left: 25%; 65 | margin-top: 20px; 66 | display: flex; 67 | justify-content: center; 68 | } 69 | 70 | #custom-prompt-input { 71 | width: 95%; 72 | background-color: rgba(0, 0, 0, 0); 73 | color: #FFFFFF; 74 | padding: 10px; 75 | border: 1px solid #FFFFFFB2; 76 | border-right: 0; 77 | border-top-left-radius: 4px; 78 | border-bottom-left-radius: 4px; 79 | } 80 | 81 | #custom-prompt-button { 82 | width: 5%; 83 | background-color: rgba(0, 0, 0, 0); 84 | border: 1px solid #FFFFFFB2; 85 | border-left: 0; 86 | border-top-right-radius: 4px; 87 | border-bottom-right-radius: 4px; 88 | } 89 | 90 | #custom-prompt-button:hover { 91 | cursor: pointer; 92 | } 93 | -------------------------------------------------------------------------------- /static/images/favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /tests/utils/mocks/ollama.ts: -------------------------------------------------------------------------------- 1 | import { MOCK_AGENT, MOCK_CONTENT_RESPONSE, MOCK_STREAMING_CONTENT_CHUNKS, establishMockAgent } from './base.js' 2 | 3 | export const OLLAMA_MOCK_HOST = 'http://127.0.0.1:41434' 4 | 5 | export let chatHistoryProvided = false 6 | 7 | export function resetOllamaMock (): void { 8 | chatHistoryProvided = false 9 | } 10 | 11 | let isOllamaMocked = false 12 | 13 | /** 14 | * @see https://github.com/ollama/ollama/blob/9446b795b58e32c8b248a76707780f4f96b6434f/docs/api.md 15 | */ 16 | export function mockOllama (): void { 17 | if (isOllamaMocked) { 18 | return 19 | } 20 | 21 | isOllamaMocked = true 22 | 23 | establishMockAgent() 24 | 25 | const pool = MOCK_AGENT.get(OLLAMA_MOCK_HOST) 26 | pool.intercept({ 27 | path: '/api/chat', 28 | method: 'POST' 29 | }).reply(200, (opts: any) => { 30 | if (typeof opts.body !== 'string') { 31 | throw new Error(`body is not a string (${typeof opts.body})`) 32 | } 33 | 34 | const body = JSON.parse(opts.body) 35 | if (body.messages.length > 1) { 36 | chatHistoryProvided = true 37 | } 38 | 39 | let response = '' 40 | if (body.stream === true) { 41 | for (let i = 0; i < MOCK_STREAMING_CONTENT_CHUNKS.length; i++) { 42 | response += JSON.stringify({ 43 | model: 'llama2', 44 | created_at: '2023-08-04T08:52:19.385406455-07:00', 45 | message: { 46 | role: 'assistant', 47 | content: MOCK_STREAMING_CONTENT_CHUNKS[i], 48 | images: null 49 | }, 50 | done: i === MOCK_STREAMING_CONTENT_CHUNKS.length - 1 51 | }) 52 | response += '\n' 53 | } 54 | } else { 55 | response += JSON.stringify({ 56 | model: 'llama2', 57 | created_at: '2023-08-04T19:22:45.499127Z', 58 | message: { 59 | role: 'assistant', 60 | content: MOCK_CONTENT_RESPONSE, 61 | images: null 62 | }, 63 | done: true 64 | }) 65 | } 66 | 67 | return response 68 | }, { 69 | headers: { 70 | 'content-type': 'application/json' 71 | } 72 | }).persist() 73 | } 74 | -------------------------------------------------------------------------------- /tests/unit/ai-providers.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-floating-promises */ 2 | import { describe, it } from 'node:test' 3 | import assert from 'node:assert' 4 | import { MistralProvider } from '../../ai-providers/mistral.js' 5 | import { OpenAiProvider } from '../../ai-providers/open-ai.js' 6 | import { AiProvider } from '../../ai-providers/provider.js' 7 | import { OllamaProvider } from '../../ai-providers/ollama.js' 8 | import { AzureProvider } from '../../ai-providers/azure.js' 9 | import { MOCK_CONTENT_RESPONSE, buildExpectedStreamBodyString } from '../utils/mocks/base.js' 10 | import { OLLAMA_MOCK_HOST } from '../utils/mocks/ollama.js' 11 | import { AZURE_DEPLOYMENT_NAME, AZURE_MOCK_HOST } from '../utils/mocks/azure.js' 12 | import { mockLlama2 } from '../utils/mocks/llama2.js' 13 | import { mockAllProviders } from '../utils/mocks/index.js' 14 | mockAllProviders() 15 | 16 | const expectedStreamBody = buildExpectedStreamBodyString() 17 | 18 | const { Llama2Provider: MockedLlamaProvider } = await mockLlama2() 19 | 20 | const providers: AiProvider[] = [ 21 | new OpenAiProvider({ model: 'gpt-3.5-turbo', apiKey: '' }), 22 | new MistralProvider({ model: 'open-mistral-7b', apiKey: '' }), 23 | new OllamaProvider({ host: OLLAMA_MOCK_HOST, model: 'some-model' }), 24 | new AzureProvider({ 25 | endpoint: AZURE_MOCK_HOST, 26 | apiKey: 'abc', 27 | deploymentName: AZURE_DEPLOYMENT_NAME, 28 | allowInsecureConnections: true 29 | }), 30 | new MockedLlamaProvider({ modelPath: '' }) 31 | ] 32 | 33 | for (const provider of providers) { 34 | describe(provider.constructor.name, () => { 35 | it('ask', async () => { 36 | const response = await provider.ask('asd') 37 | assert.strictEqual(response, MOCK_CONTENT_RESPONSE) 38 | }) 39 | 40 | it('askStream', async () => { 41 | const response = await provider.askStream('asd') 42 | const reader = response.getReader() 43 | 44 | let body = '' 45 | const decoder = new TextDecoder() 46 | while (true) { 47 | const { value, done } = await reader.read() 48 | if (done !== undefined && done) { 49 | break 50 | } 51 | 52 | body += decoder.decode(value) 53 | } 54 | 55 | assert.strictEqual(body, expectedStreamBody) 56 | }) 57 | }) 58 | } 59 | -------------------------------------------------------------------------------- /tests/types/schema.test-d.ts: -------------------------------------------------------------------------------- 1 | import { expectAssignable } from 'tsd' 2 | import { AiWarpConfig } from '../../config.js' 3 | 4 | expectAssignable({ 5 | openai: { 6 | model: 'gpt-3.5-turbo', 7 | apiKey: '' 8 | } 9 | }) 10 | 11 | expectAssignable({ 12 | openai: { 13 | model: 'gpt-4', 14 | apiKey: '' 15 | } 16 | }) 17 | 18 | expectAssignable({ 19 | mistral: { 20 | model: 'open-mistral-7b', 21 | apiKey: '' 22 | } 23 | }) 24 | 25 | expectAssignable({ 26 | aiProvider: { 27 | openai: { 28 | model: 'gpt-3.5-turbo', 29 | apiKey: '' 30 | } 31 | } 32 | }) 33 | 34 | expectAssignable({ 35 | aiProvider: { 36 | mistral: { 37 | model: 'open-mistral-7b', 38 | apiKey: '' 39 | } 40 | } 41 | }) 42 | 43 | expectAssignable({ 44 | aiProvider: { 45 | ollama: { 46 | host: '', 47 | model: 'some-model' 48 | } 49 | } 50 | }) 51 | 52 | expectAssignable({ 53 | aiProvider: { 54 | llama2: { 55 | modelPath: '/some/model.bin' 56 | } 57 | } 58 | }) 59 | 60 | expectAssignable({ 61 | $schema: './stackable.schema.json', 62 | service: { 63 | openapi: true 64 | }, 65 | watch: true, 66 | server: { 67 | hostname: '{PLT_SERVER_HOSTNAME}', 68 | port: '{PORT}', 69 | logger: { 70 | level: 'info' 71 | } 72 | }, 73 | module: '@platformatic/ai-warp', 74 | aiProvider: { 75 | openai: { 76 | model: 'gpt-3.5-turbo', 77 | apiKey: '{PLT_OPENAI_API_KEY}' 78 | } 79 | }, 80 | promptDecorators: { 81 | prefix: '', 82 | suffix: '' 83 | }, 84 | plugins: { 85 | paths: [ 86 | { 87 | path: './plugins', 88 | encapsulate: false 89 | } 90 | ] 91 | } 92 | }) 93 | 94 | expectAssignable({}) 95 | 96 | expectAssignable({ 97 | prefix: '' 98 | }) 99 | 100 | expectAssignable({ 101 | suffix: '' 102 | }) 103 | 104 | expectAssignable({ 105 | prefix: '', 106 | suffix: '' 107 | }) 108 | -------------------------------------------------------------------------------- /static/styles/chat.css: -------------------------------------------------------------------------------- 1 | #messages { 2 | width: 50%; 3 | height: 550px; 4 | max-height: 550px; 5 | margin-left: 25%; 6 | margin-top: 50px; 7 | overflow: scroll; 8 | overflow-anchor: auto; 9 | } 10 | 11 | .message { 12 | display: flex; 13 | } 14 | 15 | .message-avatar { 16 | width: 5%; 17 | padding-right: 15px; 18 | } 19 | 20 | .message-contents { 21 | width: 100%; 22 | } 23 | 24 | .message-author { 25 | margin-top: 0; 26 | font-weight: 600; 27 | } 28 | 29 | .message-error { 30 | background-color: rgba(250, 33, 33, 0.3); 31 | border: 1px solid #FA2121; 32 | border-radius: 4px; 33 | color: #FA21214D; 34 | padding: 4px 8px 4px 8px; 35 | } 36 | 37 | .message-options { 38 | width: 100%; 39 | display: inline-block; 40 | } 41 | 42 | .message-options button { 43 | border: 0; 44 | background-color: rgba(0, 0, 0, 0); 45 | cursor: pointer; 46 | } 47 | 48 | .response-index-selector { 49 | display: flex; 50 | float: left; 51 | } 52 | 53 | .message-options-right { 54 | float: right; 55 | } 56 | 57 | .response-index-selector p { 58 | margin: 0; 59 | } 60 | 61 | .submit-prompt-edit-button { 62 | padding: 8px !important; 63 | background: white !important; 64 | border: 1px solid white !important; 65 | border-radius: 4px; 66 | color: #00050B; 67 | margin-right: 8px; 68 | } 69 | 70 | .cancel-prompt-edit-button { 71 | padding: 8px !important; 72 | border: 1px solid white !important; 73 | border-radius: 4px; 74 | color: white; 75 | } 76 | 77 | #prompt { 78 | width: 50%; 79 | margin-left: 25%; 80 | margin-top: 50px; 81 | display: flex; 82 | justify-content: center; 83 | } 84 | 85 | #prompt-input { 86 | width: 95%; 87 | background-color: rgba(0, 0, 0, 0); 88 | color: #FFFFFF; 89 | padding: 10px; 90 | border: 1px solid #FFFFFFB2; 91 | border-right: 0; 92 | border-top-left-radius: 4px; 93 | border-bottom-left-radius: 4px; 94 | } 95 | 96 | #prompt-button { 97 | width: 5%; 98 | background-color: rgba(0, 0, 0, 0); 99 | border: 1px solid #FFFFFFB2; 100 | border-left: 0; 101 | border-top-right-radius: 4px; 102 | border-bottom-right-radius: 4px; 103 | } 104 | 105 | #prompt-button:hover { 106 | cursor: pointer; 107 | } 108 | 109 | #prompt-button:disabled { 110 | cursor: not-allowed; 111 | } 112 | 113 | #prompt-button:disabled img { 114 | opacity: 0.2; 115 | } 116 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@platformatic/ai-warp", 3 | "version": "0.5.1", 4 | "main": "dist/index.js", 5 | "type": "module", 6 | "bin": { 7 | "create-ai-warp": "./dist/cli/create.js", 8 | "start-ai-warp": "./dist/cli/start.js" 9 | }, 10 | "scripts": { 11 | "create": "node ./dist/cli/create.js", 12 | "start": "node ./dist/cli/start.js -c ./ai-warp-app/platformatic.json", 13 | "build": "tsc --build && cp -r ./static ./dist/ && cp config.d.ts dist/", 14 | "build:config": "npm run build && node ./dist/lib/schema.js --dump-schema > schema.json && json2ts > config.d.ts < schema.json", 15 | "prepare": "npm run build:config", 16 | "clean": "rm -fr ./dist", 17 | "lint": "ts-standard | snazzy", 18 | "lint:fix": "ts-standard --fix | snazzy", 19 | "lint-md": "markdownlint-cli2 .", 20 | "lint-md:fix": "markdownlint-cli2 --fix .", 21 | "test": "npm run test:unit && npm run test:e2e && npm run test:types", 22 | "test:unit": "npm run build && node --test --test-reporter=@reporters/github --test-reporter-destination=stdout --test-reporter=spec --test-reporter-destination=stdout --import=tsx --test-concurrency=1 ./tests/unit/*", 23 | "test:e2e": "node --test --test-reporter=@reporters/github --test-reporter-destination=stdout --test-reporter=spec --test-reporter-destination=stdout --import=tsx --test-concurrency=1 ./tests/e2e/*", 24 | "test:types": "tsd" 25 | }, 26 | "engines": { 27 | "node": ">=20.16.0" 28 | }, 29 | "devDependencies": { 30 | "@reporters/github": "^1.7.2", 31 | "fastify": "^5.2.0", 32 | "markdownlint-cli2": "^0.18.0", 33 | "node-llama-cpp": "^2.8.16", 34 | "snazzy": "^9.0.0", 35 | "ts-standard": "^12.0.2", 36 | "tsd": "^0.32.0", 37 | "tsx": "^4.19.2", 38 | "typescript": "^5.7.2" 39 | }, 40 | "dependencies": { 41 | "@azure/openai": "^1.0.0-beta.12", 42 | "@fastify/error": "^4.0.0", 43 | "@fastify/rate-limit": "^10.2.1", 44 | "@fastify/static": "^8.0.3", 45 | "@fastify/type-provider-typebox": "^5.1.0", 46 | "@platformatic/config": "^2.1.1", 47 | "@platformatic/generators": "^2.1.1", 48 | "@platformatic/mistral-client": "^0.1.0", 49 | "@platformatic/service": "^2.24.0", 50 | "esmock": "^2.6.9", 51 | "fast-json-stringify": "^6.0.0", 52 | "fastify-user": "^1.4.0", 53 | "json-schema-to-typescript": "^15.0.3", 54 | "ollama": "^0.5.11", 55 | "openai": "^4.76.3" 56 | }, 57 | "license": "Apache-2.0", 58 | "overrides": { 59 | "minimatch": "^10.0.0" 60 | }, 61 | "tsd": { 62 | "directory": "tests/types" 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /index.ts: -------------------------------------------------------------------------------- 1 | import { join } from 'node:path' 2 | import type { FastifyInstance } from 'fastify' 3 | import { app as platformaticService, Stackable, buildStackable as serviceBuildStackable } from '@platformatic/service' 4 | import { ConfigManager } from '@platformatic/config' 5 | import type { StackableInterface } from '@platformatic/config' 6 | import fastifyUser from 'fastify-user' 7 | import fastifyStatic from '@fastify/static' 8 | import { schema } from './lib/schema.js' 9 | import { Generator } from './lib/generator.js' 10 | import { AiWarpConfig } from './config.js' 11 | import warpPlugin from './plugins/warp.js' 12 | import authPlugin from './plugins/auth.js' 13 | import apiPlugin from './plugins/api.js' 14 | import rateLimitPlugin from './plugins/rate-limiting.js' 15 | 16 | export interface AiWarpMixin { 17 | platformatic: { 18 | configManager: ConfigManager 19 | config: AiWarpConfig 20 | } 21 | } 22 | 23 | type AiGenerator = new () => Generator 24 | 25 | async function buildStackable (opts: { config: string }): Promise { 26 | // eslint-disable-next-line 27 | return await serviceBuildStackable(opts, stackable) 28 | } 29 | 30 | const stackable: Stackable = { 31 | async app (app, opts) { 32 | const fastify = app as unknown as FastifyInstance & AiWarpMixin 33 | const { config } = fastify.platformatic 34 | // @ts-expect-error 35 | await fastify.register(fastifyUser as any, config.auth) 36 | await fastify.register(authPlugin, opts) 37 | 38 | if (config.showAiWarpHomepage !== undefined && config.showAiWarpHomepage) { 39 | await fastify.register(fastifyStatic, { 40 | root: join(import.meta.dirname, 'static'), 41 | wildcard: false 42 | }) 43 | } 44 | 45 | await fastify.register(platformaticService, opts) 46 | 47 | await fastify.register(warpPlugin, opts) // needs to be registered here for fastify.ai to be decorated 48 | 49 | await fastify.register(rateLimitPlugin, opts) 50 | await fastify.register(apiPlugin, opts) 51 | }, 52 | configType: 'ai-warp-app', 53 | schema, 54 | Generator, 55 | configManagerConfig: { 56 | schema, 57 | envWhitelist: ['PORT', 'HOSTNAME'], 58 | allowToWatch: ['.env'], 59 | schemaOptions: { 60 | useDefaults: true, 61 | coerceTypes: true, 62 | allErrors: true, 63 | strict: false 64 | }, 65 | async transformConfig () {} 66 | }, 67 | buildStackable 68 | } 69 | 70 | // break Fastify encapsulation 71 | // @ts-expect-error 72 | stackable.app[Symbol.for('skip-override')] = true 73 | 74 | export default stackable 75 | export { Generator, schema, buildStackable } 76 | -------------------------------------------------------------------------------- /tests/types/fastify.test-d.ts: -------------------------------------------------------------------------------- 1 | import { ReadableStream } from 'node:stream/web' 2 | import { FastifyInstance, FastifyRequest } from 'fastify' 3 | import { errorResponseBuilderContext } from '@fastify/rate-limit' 4 | import { expectAssignable } from 'tsd' 5 | import '../../index.js' 6 | 7 | expectAssignable(async (_: FastifyRequest, _2: string) => '') 8 | 9 | expectAssignable(async (_: FastifyRequest, _2: string) => new ReadableStream()) 10 | 11 | expectAssignable((_: FastifyRequest, _2: string) => '') 12 | expectAssignable(async (_: FastifyRequest, _2: string) => '') 13 | 14 | expectAssignable((_: FastifyRequest, _2: string) => '') 15 | expectAssignable(async (_: FastifyRequest, _2: string) => '') 16 | 17 | expectAssignable((_: FastifyRequest, _2: string) => 0) 18 | expectAssignable(async (_: FastifyRequest, _2: string) => 0) 19 | 20 | expectAssignable((_: FastifyRequest, _2: string) => true) 21 | expectAssignable(async (_: FastifyRequest, _2: string) => true) 22 | 23 | expectAssignable((_: FastifyRequest, _2: string) => {}) 24 | expectAssignable(async (_: FastifyRequest, _2: string) => {}) 25 | 26 | expectAssignable((_: FastifyRequest) => '') 27 | expectAssignable((_: FastifyRequest) => 0) 28 | expectAssignable(async (_: FastifyRequest) => '') 29 | expectAssignable(async (_: FastifyRequest) => 0) 30 | 31 | expectAssignable((_: FastifyRequest, _2: errorResponseBuilderContext) => { return {} }) 32 | 33 | expectAssignable((_: FastifyRequest, _2: string) => {}) 34 | expectAssignable(async (_: FastifyRequest, _2: string) => {}) 35 | 36 | expectAssignable((_: FastifyRequest, _2: string) => {}) 37 | expectAssignable(async (_: FastifyRequest, _2: string) => {}) 38 | -------------------------------------------------------------------------------- /tests/utils/mocks/mistral.ts: -------------------------------------------------------------------------------- 1 | import { MOCK_AGENT, MOCK_CONTENT_RESPONSE, MOCK_STREAMING_CONTENT_CHUNKS, establishMockAgent } from './base.js' 2 | 3 | export let chatHistoryProvided = false 4 | 5 | export function resetMistralMock (): void { 6 | chatHistoryProvided = false 7 | } 8 | 9 | let isMistralMocked = false 10 | 11 | /** 12 | * Mock Mistral's rest api 13 | * @see https://docs.mistral.ai/api/#operation/createChatCompletion 14 | */ 15 | export function mockMistralApi (): void { 16 | if (isMistralMocked) { 17 | return 18 | } 19 | 20 | isMistralMocked = true 21 | 22 | establishMockAgent() 23 | 24 | const pool = MOCK_AGENT.get('https://api.mistral.ai') 25 | pool.intercept({ 26 | path: '/v1/chat/completions', 27 | method: 'POST' 28 | }).reply(200, (opts: any) => { 29 | if (typeof opts.body !== 'string') { 30 | throw new Error(`body is not a string (${typeof opts.body})`) 31 | } 32 | 33 | const body = JSON.parse(opts.body) 34 | if (body.messages.length > 1) { 35 | chatHistoryProvided = true 36 | } 37 | 38 | let response = '' 39 | if (body.stream === true) { 40 | for (let i = 0; i < MOCK_STREAMING_CONTENT_CHUNKS.length; i++) { 41 | response += 'data: ' 42 | response += JSON.stringify({ 43 | id: 'cmpl-e5cc70bb28c444948073e77776eb30ef', 44 | object: 'chat.completion.chunk', 45 | created: 1694268190, 46 | model: 'mistral-small-latest', 47 | choices: [{ 48 | index: 0, 49 | delta: { 50 | role: 'assistant', 51 | content: MOCK_STREAMING_CONTENT_CHUNKS[i] 52 | }, 53 | logprobs: null, 54 | finish_reason: i === MOCK_STREAMING_CONTENT_CHUNKS.length ? 'stop' : null 55 | }] 56 | }) 57 | response += '\n\n' 58 | } 59 | response += 'data: [DONE]\n\n' 60 | } else { 61 | response += JSON.stringify({ 62 | id: 'cmpl-e5cc70bb28c444948073e77776eb30ef', 63 | object: 'chat.completion', 64 | created: new Date().getTime() / 1000, 65 | model: 'mistral-small-latest', 66 | choices: [{ 67 | index: 0, 68 | message: { 69 | role: 'assistant', 70 | content: MOCK_CONTENT_RESPONSE 71 | }, 72 | logprobs: null, 73 | finish_reason: 'stop' 74 | }], 75 | usage: { 76 | prompt_tokens: 1, 77 | completion_tokens: 1, 78 | total_tokens: 2 79 | } 80 | }) 81 | } 82 | 83 | return response 84 | }, { 85 | headers: { 86 | 'content-type': 'application/json' 87 | } 88 | }).persist() 89 | } 90 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Steps for downloading and setting up AI Warp for local development. 4 | 5 | ## Steps 6 | 7 | 1. Fork the repository. 8 | 9 | 2. Clone your fork using SSH, Github CLI, or HTTPS. 10 | 11 | ```bash 12 | git clone git@github.com:/ai-warp.git # SSH 13 | git clone https://github.com//ai-warp.git # HTTPS 14 | gh repo clone /ai-warp # GitHub CLI 15 | ``` 16 | 17 | 3. Install [Node.js](https://nodejs.org/). 18 | 19 | 4. Install dependencies. 20 | 21 | ```bash 22 | npm install 23 | ``` 24 | 25 | 5. Build. 26 | 27 | ```bash 28 | npm run build 29 | ``` 30 | 31 | 6. Generate the test app. 32 | 33 | ```bash 34 | npm run create 35 | ``` 36 | 37 | 7. Configure the test app's `platformatic.json` to your liking. By default, it 38 | is located at `ai-warp-app/platformatic.json`. **Note: this will be overwrited 39 | every time you generate the test app.** 40 | 41 | 8. Start the test app. From the `app-warp-ai` folder, run: 42 | 43 | ```bash 44 | node ../dist/cli/start.js 45 | ``` 46 | 47 | ### Testing a model with OpenAI 48 | 49 | To test a remote model with with OpenAI, you can use the following to 50 | download the model we used for testing: 51 | 52 | ```json 53 | "aiProvider": { 54 | "openai": { 55 | "model": "gpt-3.5-turbo", 56 | "apiKey": "{PLT_OPENAI_API_KEY}" 57 | } 58 | } 59 | ``` 60 | 61 | Make sure to add your OpenAI api key as `PLT_OPENAI_API_KEY` in your `.env` file. 62 | 63 | ### Testing a local model with llama2 64 | 65 | To test a local model with with llama2, you can use the following to 66 | download the model we used for testing: 67 | 68 | ```bash 69 | curl -L -O https://huggingface.co/TheBloke/Mistral-7B-Instruct-v0.2-GGUF/resolve/main/mistral-7b-instruct-v0.2.Q8_0.gguf 70 | ``` 71 | 72 | Then, in your `platformatic.json` file, add: 73 | 74 | ```json 75 | "aiProvider": { 76 | "llama2": { 77 | "modelPath": "./mistral-7b-instruct-v0.2.Q8_0.gguf" 78 | } 79 | }, 80 | ``` 81 | 82 | ## Important Notes 83 | 84 | * AI Warp needs to be rebuilt for any code change to take affect in your test 85 | app. This includes schema changes. 86 | 87 | ## Noteable Commands 88 | 89 | * `npm run build` - Build the app. 90 | * `npm run build:config` - Rebuild the config. 91 | * `npm run lint:fix` - Fix all formatting issues and console log any linting 92 | issues that need to be fixed in code. 93 | * `npm run test` - Run Unit, E2E, and Type tests. 94 | 95 | ## Additional Resources 96 | 97 | * [Use Stackables to build Platformatic applications](https://docs.platformatic.dev/docs/guides/applications-with-stackables) 98 | -------------------------------------------------------------------------------- /tests/utils/mocks/open-ai.ts: -------------------------------------------------------------------------------- 1 | import { MOCK_AGENT, MOCK_CONTENT_RESPONSE, MOCK_STREAMING_CONTENT_CHUNKS, establishMockAgent } from './base.js' 2 | 3 | export let chatHistoryProvided = false 4 | 5 | export function resetOpenAiMock (): void { 6 | chatHistoryProvided = false 7 | } 8 | 9 | let isOpenAiMocked = false 10 | 11 | /** 12 | * Mock OpenAI's rest api 13 | * @see https://platform.openai.com/docs/api-reference/chat 14 | */ 15 | export function mockOpenAiApi (): void { 16 | if (isOpenAiMocked) { 17 | return 18 | } 19 | 20 | isOpenAiMocked = true 21 | 22 | establishMockAgent() 23 | 24 | const pool = MOCK_AGENT.get('https://api.openai.com') 25 | pool.intercept({ 26 | path: '/v1/chat/completions', 27 | method: 'POST' 28 | }).reply(200, (opts: any) => { 29 | if (typeof opts.body !== 'string') { 30 | throw new Error(`body is not a string (${typeof opts.body})`) 31 | } 32 | 33 | const body = JSON.parse(opts.body) 34 | if (body.messages.length > 1) { 35 | chatHistoryProvided = true 36 | } 37 | 38 | let response = '' 39 | if (body.stream === true) { 40 | for (let i = 0; i < MOCK_STREAMING_CONTENT_CHUNKS.length; i++) { 41 | response += 'data: ' 42 | response += JSON.stringify({ 43 | id: 'chatcmpl-123', 44 | object: 'chat.completion.chunk', 45 | created: 1694268190, 46 | model: 'gpt-3.5-turbo-0125', 47 | system_fingerprint: 'fp_44709d6fcb', 48 | choices: [{ 49 | index: 0, 50 | delta: { 51 | role: 'assistant', 52 | content: MOCK_STREAMING_CONTENT_CHUNKS[i] 53 | }, 54 | logprobs: null, 55 | finish_reason: i === MOCK_STREAMING_CONTENT_CHUNKS.length ? 'stop' : null 56 | }] 57 | }) 58 | response += '\n\n' 59 | } 60 | response += 'data: [DONE]\n\n' 61 | } else { 62 | response += JSON.stringify({ 63 | id: 'chatcmpl-123', 64 | object: 'chat.completion', 65 | created: new Date().getTime() / 1000, 66 | model: 'gpt-3.5-turbo-0125', 67 | system_fingerprint: 'fp_fp_44709d6fcb', 68 | choices: [{ 69 | index: 0, 70 | message: { 71 | role: 'assistant', 72 | content: MOCK_CONTENT_RESPONSE 73 | }, 74 | logprobs: null, 75 | finish_reason: 'stop' 76 | }], 77 | usage: { 78 | prompt_tokens: 1, 79 | completion_tokens: 1, 80 | total_tokens: 2 81 | } 82 | }) 83 | } 84 | 85 | return response 86 | }, { 87 | headers: { 88 | 'content-type': 'application/json' 89 | } 90 | }).persist() 91 | } 92 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | setup-node-modules: 11 | runs-on: ubuntu-latest 12 | timeout-minutes: 15 13 | steps: 14 | - name: Git Checkout 15 | uses: actions/checkout@09d2acae674a48949e3602304ab46fd20ae0c42f 16 | 17 | - name: Setup Node 18 | uses: actions/setup-node@v4 19 | with: 20 | node-version: 20 21 | cache: 'npm' 22 | 23 | - name: Install Dependencies 24 | run: npm install 25 | 26 | lint: 27 | name: Linting 28 | runs-on: ubuntu-latest 29 | needs: setup-node-modules 30 | steps: 31 | - name: Git Checkout 32 | uses: actions/checkout@09d2acae674a48949e3602304ab46fd20ae0c42f 33 | 34 | - name: Setup Node 35 | uses: actions/setup-node@v4 36 | with: 37 | node-version: 20 38 | cache: 'npm' 39 | 40 | - name: Install Dependencies 41 | run: npm install 42 | 43 | - name: Run Linting 44 | run: npm run lint 45 | 46 | unit-tests: 47 | name: Unit Tests 48 | runs-on: ubuntu-latest 49 | needs: setup-node-modules 50 | steps: 51 | - name: Git Checkout 52 | uses: actions/checkout@09d2acae674a48949e3602304ab46fd20ae0c42f 53 | 54 | - name: Setup Node 55 | uses: actions/setup-node@v4 56 | with: 57 | node-version: 20 58 | cache: 'npm' 59 | 60 | - name: Install Dependencies 61 | run: npm install 62 | 63 | - name: Run Tests 64 | run: npm run test:unit 65 | 66 | e2e-tests: 67 | name: E2E Tests 68 | runs-on: ubuntu-latest 69 | needs: setup-node-modules 70 | steps: 71 | - name: Git Checkout 72 | uses: actions/checkout@09d2acae674a48949e3602304ab46fd20ae0c42f 73 | 74 | - name: Setup Node 75 | uses: actions/setup-node@v4 76 | with: 77 | node-version: 20 78 | cache: 'npm' 79 | 80 | - name: Install Dependencies 81 | run: npm install 82 | 83 | - name: Run Tests 84 | run: npm run test:e2e 85 | 86 | type-tests: 87 | name: Type Tests 88 | runs-on: ubuntu-latest 89 | needs: setup-node-modules 90 | steps: 91 | - name: Git Checkout 92 | uses: actions/checkout@09d2acae674a48949e3602304ab46fd20ae0c42f 93 | 94 | - name: Setup Node 95 | uses: actions/setup-node@v4 96 | with: 97 | node-version: 20 98 | cache: 'npm' 99 | 100 | - name: Install Dependencies 101 | run: npm install 102 | 103 | - name: Run Tests 104 | run: npm run test:types 105 | -------------------------------------------------------------------------------- /plugins/api.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line 2 | /// 3 | import { FastifyError } from 'fastify' 4 | import { FastifyPluginAsyncTypebox, Type } from '@fastify/type-provider-typebox' 5 | import createError from '@fastify/error' 6 | 7 | function isAFastifyError (object: object): object is FastifyError { 8 | return 'code' in object && 'name' in object 9 | } 10 | 11 | const InternalServerError = createError('INTERNAL_SERVER_ERROR', 'Internal Server Error', 500) 12 | 13 | const plugin: FastifyPluginAsyncTypebox = async (fastify) => { 14 | fastify.route({ 15 | url: '/api/v1/prompt', 16 | method: 'POST', 17 | schema: { 18 | operationId: 'prompt', 19 | body: Type.Object({ 20 | prompt: Type.String(), 21 | chatHistory: Type.Optional(Type.Array(Type.Object({ 22 | prompt: Type.String(), 23 | response: Type.String() 24 | }))) 25 | }), 26 | response: { 27 | 200: Type.Object({ 28 | response: Type.String() 29 | }), 30 | default: Type.Object({ 31 | code: Type.Optional(Type.String()), 32 | message: Type.String() 33 | }) 34 | } 35 | }, 36 | handler: async (request) => { 37 | try { 38 | const { prompt, chatHistory } = request.body 39 | const response = await fastify.ai.warp(request, prompt, chatHistory) 40 | 41 | return { response } 42 | } catch (exception) { 43 | if (exception instanceof Object && isAFastifyError(exception)) { 44 | return exception 45 | } else { 46 | const err = new InternalServerError() 47 | err.cause = exception 48 | throw err 49 | } 50 | } 51 | } 52 | }) 53 | 54 | fastify.route({ 55 | url: '/api/v1/stream', 56 | method: 'POST', 57 | schema: { 58 | operationId: 'stream', 59 | produces: ['text/event-stream'], 60 | body: Type.Object({ 61 | prompt: Type.String(), 62 | chatHistory: Type.Optional(Type.Array(Type.Object({ 63 | prompt: Type.String(), 64 | response: Type.String() 65 | }))) 66 | }) 67 | }, 68 | handler: async (request, reply) => { 69 | try { 70 | const { prompt, chatHistory } = request.body 71 | 72 | const response = await fastify.ai.warpStream(request, prompt, chatHistory) 73 | // eslint-disable-next-line @typescript-eslint/no-floating-promises 74 | reply.header('content-type', 'text/event-stream') 75 | 76 | return response 77 | } catch (exception) { 78 | if (exception instanceof Object && isAFastifyError(exception)) { 79 | return exception 80 | } else { 81 | const err = new InternalServerError() 82 | err.cause = exception 83 | throw err 84 | } 85 | } 86 | } 87 | }) 88 | } 89 | 90 | export default plugin 91 | -------------------------------------------------------------------------------- /ai-providers/ollama.ts: -------------------------------------------------------------------------------- 1 | import { ReadableStream, UnderlyingByteSource, ReadableByteStreamController } from 'stream/web' 2 | import { Ollama, ChatResponse, Message } from 'ollama' 3 | import type { AbortableAsyncIterator } from 'ollama/src/utils.js' 4 | import { AiProvider, ChatHistory, StreamChunkCallback } from './provider.js' 5 | import { AiStreamEvent, encodeEvent } from './event.js' 6 | 7 | type OllamaStreamResponse = AbortableAsyncIterator 8 | 9 | class OllamaByteSource implements UnderlyingByteSource { 10 | type: 'bytes' = 'bytes' 11 | response: OllamaStreamResponse 12 | chunkCallback?: StreamChunkCallback 13 | 14 | constructor (response: OllamaStreamResponse, chunkCallback?: StreamChunkCallback) { 15 | this.response = response 16 | this.chunkCallback = chunkCallback 17 | } 18 | 19 | async pull (controller: ReadableByteStreamController): Promise { 20 | for await (const { done, message } of this.response) { 21 | let response = message.content 22 | if (this.chunkCallback !== undefined) { 23 | response = await this.chunkCallback(response) 24 | } 25 | 26 | const eventData: AiStreamEvent = { 27 | event: 'content', 28 | data: { 29 | response 30 | } 31 | } 32 | controller.enqueue(encodeEvent(eventData)) 33 | 34 | if (done !== undefined && done) { 35 | controller.close() 36 | return 37 | } 38 | } 39 | } 40 | } 41 | 42 | interface OllamaProviderCtorOptions { 43 | host: string 44 | model: string 45 | } 46 | 47 | export class OllamaProvider implements AiProvider { 48 | model: string 49 | client: Ollama 50 | 51 | constructor ({ host, model }: OllamaProviderCtorOptions) { 52 | this.model = model 53 | this.client = new Ollama({ host }) 54 | } 55 | 56 | async ask (prompt: string, chatHistory?: ChatHistory): Promise { 57 | const response = await this.client.chat({ 58 | model: this.model, 59 | messages: [ 60 | ...this.chatHistoryToMessages(chatHistory), 61 | { role: 'user', content: prompt } 62 | ] 63 | }) 64 | 65 | return response.message.content 66 | } 67 | 68 | async askStream (prompt: string, chunkCallback?: StreamChunkCallback, chatHistory?: ChatHistory): Promise { 69 | const response = await this.client.chat({ 70 | model: this.model, 71 | messages: [ 72 | ...this.chatHistoryToMessages(chatHistory), 73 | { role: 'user', content: prompt } 74 | ], 75 | stream: true 76 | }) 77 | 78 | // @ts-expect-error polyfill type mismatch 79 | return new ReadableStream(new OllamaByteSource(response, chunkCallback)) 80 | } 81 | 82 | private chatHistoryToMessages (chatHistory?: ChatHistory): Message[] { 83 | if (chatHistory === undefined) { 84 | return [] 85 | } 86 | 87 | const messages: Message[] = [] 88 | for (const previousInteraction of chatHistory) { 89 | messages.push({ role: 'user', content: previousInteraction.prompt }) 90 | messages.push({ role: 'assistant', content: previousInteraction.response }) 91 | } 92 | 93 | return messages 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /plugins/warp.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line 2 | /// 3 | import { FastifyLoggerInstance } from 'fastify' 4 | import fastifyPlugin from 'fastify-plugin' 5 | import { AiProvider, StreamChunkCallback } from '../ai-providers/provider.js' 6 | import { AiWarpConfig } from '../config.js' 7 | import createError from '@fastify/error' 8 | 9 | const UnknownAiProviderError = createError('UNKNOWN_AI_PROVIDER', 'Unknown AI Provider') 10 | 11 | async function build (aiProvider: AiWarpConfig['aiProvider'], logger: FastifyLoggerInstance): Promise { 12 | if ('openai' in aiProvider) { 13 | const { OpenAiProvider } = await import('../ai-providers/open-ai.js') 14 | return new OpenAiProvider(aiProvider.openai) 15 | } else if ('mistral' in aiProvider) { 16 | const { MistralProvider } = await import('../ai-providers/mistral.js') 17 | return new MistralProvider(aiProvider.mistral) 18 | } else if ('ollama' in aiProvider) { 19 | const { OllamaProvider } = await import('../ai-providers/ollama.js') 20 | return new OllamaProvider(aiProvider.ollama) 21 | } else if ('azure' in aiProvider) { 22 | const { AzureProvider } = await import('../ai-providers/azure.js') 23 | return new AzureProvider(aiProvider.azure) 24 | } else if ('llama2' in aiProvider) { 25 | const { Llama2Provider } = await import('../ai-providers/llama2.js') 26 | return new Llama2Provider({ 27 | ...aiProvider.llama2, 28 | logger 29 | }) 30 | } else { 31 | throw new UnknownAiProviderError() 32 | } 33 | } 34 | 35 | export default fastifyPlugin(async (fastify) => { 36 | const { config } = fastify.platformatic 37 | const provider = await build(config.aiProvider, fastify.log) 38 | 39 | fastify.decorate('ai', { 40 | warp: async (request, prompt, chatHistory) => { 41 | let decoratedPrompt = prompt 42 | if (config.promptDecorators !== undefined) { 43 | const { prefix, suffix } = config.promptDecorators 44 | decoratedPrompt = (prefix ?? '') + decoratedPrompt + (suffix ?? '') 45 | } 46 | 47 | let response = await provider.ask(decoratedPrompt, chatHistory) 48 | if (fastify.ai.preResponseCallback !== undefined) { 49 | response = await fastify.ai.preResponseCallback(request, response) ?? response 50 | } 51 | 52 | return response 53 | }, 54 | warpStream: async (request, prompt, chatHistory) => { 55 | let decoratedPrompt = prompt 56 | if (config.promptDecorators !== undefined) { 57 | const { prefix, suffix } = config.promptDecorators 58 | decoratedPrompt = (prefix ?? '') + decoratedPrompt + (suffix ?? '') 59 | } 60 | 61 | let chunkCallback: StreamChunkCallback | undefined 62 | if (fastify.ai.preResponseChunkCallback !== undefined) { 63 | chunkCallback = async (response) => { 64 | if (fastify.ai.preResponseChunkCallback === undefined) { 65 | return response 66 | } 67 | return await fastify.ai.preResponseChunkCallback(request, response) ?? response 68 | } 69 | } 70 | 71 | const response = await provider.askStream(decoratedPrompt, chunkCallback, chatHistory) 72 | return response 73 | }, 74 | rateLimiting: {} 75 | }) 76 | }) 77 | -------------------------------------------------------------------------------- /tests/utils/mocks/azure.ts: -------------------------------------------------------------------------------- 1 | import { Server, createServer } from 'node:http' 2 | import { MOCK_CONTENT_RESPONSE, MOCK_STREAMING_CONTENT_CHUNKS } from './base.js' 3 | 4 | export const AZURE_MOCK_HOST = 'http://127.0.0.1:41435' 5 | 6 | export const AZURE_DEPLOYMENT_NAME = 'some-deployment' 7 | 8 | export let chatHistoryProvided = false 9 | 10 | export function resetAzureMock (): void { 11 | chatHistoryProvided = false 12 | } 13 | 14 | /** 15 | * @see https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions 16 | */ 17 | export function mockAzure (): Server { 18 | // The Azure client doesn't use undici's fetch and there's no option to pass 19 | // it in like the other providers' clients unfortunately, so let's create an 20 | // actual server 21 | const server = createServer((req, res) => { 22 | if (req.url !== '/openai/deployments/some-deployment/chat/completions?api-version=2024-03-01-preview') { 23 | res.end() 24 | throw new Error(`unsupported url or api version: ${req.url ?? ''}`) 25 | } 26 | 27 | let bodyString = '' 28 | req.on('data', (chunk: string) => { 29 | bodyString += chunk 30 | }) 31 | req.on('end', () => { 32 | const body: { stream: boolean, messages: unknown[] } = JSON.parse(bodyString) 33 | 34 | if (body.messages.length > 1) { 35 | chatHistoryProvided = true 36 | } 37 | 38 | if (body.stream) { 39 | res.setHeader('content-type', 'text/event-stream') 40 | 41 | for (let i = 0; i < MOCK_STREAMING_CONTENT_CHUNKS.length; i++) { 42 | res.write('data: ') 43 | res.write(JSON.stringify({ 44 | id: 'chatcmpl-6v7mkQj980V1yBec6ETrKPRqFjNw9', 45 | object: 'chat.completion', 46 | created: 1679072642, 47 | model: 'gpt-35-turbo', 48 | usage: { 49 | prompt_tokens: 58, 50 | completion_tokens: 68, 51 | total_tokens: 126 52 | }, 53 | choices: [ 54 | { 55 | delta: { 56 | role: 'assistant', 57 | content: MOCK_STREAMING_CONTENT_CHUNKS[i] 58 | }, 59 | finish_reason: i === MOCK_STREAMING_CONTENT_CHUNKS.length ? 'stop' : null, 60 | index: 0 61 | } 62 | ] 63 | })) 64 | res.write('\n\n') 65 | } 66 | res.write('data: [DONE]\n\n') 67 | } else { 68 | res.setHeader('content-type', 'application/json') 69 | res.write(JSON.stringify({ 70 | id: 'chatcmpl-6v7mkQj980V1yBec6ETrKPRqFjNw9', 71 | object: 'chat.completion', 72 | created: 1679072642, 73 | model: 'gpt-35-turbo', 74 | usage: { 75 | prompt_tokens: 58, 76 | completion_tokens: 68, 77 | total_tokens: 126 78 | }, 79 | choices: [ 80 | { 81 | message: { 82 | role: 'assistant', 83 | content: MOCK_CONTENT_RESPONSE 84 | }, 85 | finish_reason: 'stop', 86 | index: 0 87 | } 88 | ] 89 | })) 90 | } 91 | 92 | res.end() 93 | }) 94 | }) 95 | server.listen(41435) 96 | 97 | return server 98 | } 99 | -------------------------------------------------------------------------------- /ai-providers/mistral.ts: -------------------------------------------------------------------------------- 1 | import { ReadableStream, UnderlyingByteSource, ReadableByteStreamController } from 'node:stream/web' 2 | import MistralClient, { ChatCompletionResponseChunk } from '@platformatic/mistral-client' 3 | import { AiProvider, ChatHistory, NoContentError, StreamChunkCallback } from './provider.js' 4 | import { AiStreamEvent, encodeEvent } from './event.js' 5 | 6 | type MistralStreamResponse = AsyncGenerator 7 | 8 | class MistralByteSource implements UnderlyingByteSource { 9 | type: 'bytes' = 'bytes' 10 | response: MistralStreamResponse 11 | chunkCallback?: StreamChunkCallback 12 | 13 | constructor (response: MistralStreamResponse, chunkCallback?: StreamChunkCallback) { 14 | this.response = response 15 | this.chunkCallback = chunkCallback 16 | } 17 | 18 | async pull (controller: ReadableByteStreamController): Promise { 19 | const { done, value } = await this.response.next() 20 | if (done !== undefined && done) { 21 | controller.close() 22 | return 23 | } 24 | 25 | if (value.choices.length === 0) { 26 | const error = new NoContentError('Mistral (Stream)') 27 | 28 | const eventData: AiStreamEvent = { 29 | event: 'error', 30 | data: error 31 | } 32 | controller.enqueue(encodeEvent(eventData)) 33 | controller.close() 34 | 35 | return 36 | } 37 | 38 | const { content } = value.choices[0].delta 39 | 40 | let response = content ?? '' 41 | if (this.chunkCallback !== undefined) { 42 | response = await this.chunkCallback(response) 43 | } 44 | 45 | const eventData: AiStreamEvent = { 46 | event: 'content', 47 | data: { 48 | response 49 | } 50 | } 51 | controller.enqueue(encodeEvent(eventData)) 52 | } 53 | } 54 | 55 | interface MistralProviderCtorOptions { 56 | model: string 57 | apiKey: string 58 | } 59 | 60 | export class MistralProvider implements AiProvider { 61 | model: string 62 | client: MistralClient 63 | 64 | constructor ({ model, apiKey }: MistralProviderCtorOptions) { 65 | this.model = model 66 | this.client = new MistralClient(apiKey) 67 | } 68 | 69 | async ask (prompt: string, chatHistory?: ChatHistory): Promise { 70 | const response = await this.client.chat({ 71 | model: this.model, 72 | messages: [ 73 | ...this.chatHistoryToMessages(chatHistory), 74 | { role: 'user', content: prompt } 75 | ] 76 | }) 77 | 78 | if (response.choices.length === 0) { 79 | throw new NoContentError('Mistral') 80 | } 81 | 82 | return response.choices[0].message.content 83 | } 84 | 85 | async askStream (prompt: string, chunkCallback?: StreamChunkCallback, chatHistory?: ChatHistory): Promise { 86 | const response = this.client.chatStream({ 87 | model: this.model, 88 | messages: [ 89 | ...this.chatHistoryToMessages(chatHistory), 90 | { role: 'user', content: prompt } 91 | ] 92 | }) 93 | return new ReadableStream(new MistralByteSource(response, chunkCallback)) 94 | } 95 | 96 | private chatHistoryToMessages (chatHistory?: ChatHistory): Array<{ role: string, content: string }> { 97 | if (chatHistory === undefined) { 98 | return [] 99 | } 100 | 101 | const messages: Array<{ role: string, content: string }> = [] 102 | for (const previousInteraction of chatHistory) { 103 | messages.push({ role: 'user', content: previousInteraction.prompt }) 104 | messages.push({ role: 'assistant', content: previousInteraction.response }) 105 | } 106 | 107 | return messages 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /plugins/rate-limiting.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line 2 | /// 3 | import { FastifyInstance } from 'fastify' 4 | import createError from '@fastify/error' 5 | import fastifyPlugin from 'fastify-plugin' 6 | import fastifyRateLimit from '@fastify/rate-limit' 7 | import { AiWarpConfig } from '../config.js' 8 | 9 | interface RateLimitMax { 10 | // One claim to many values & maxes 11 | values: Record 12 | } 13 | 14 | function buildMaxByClaimLookupTable (config: AiWarpConfig['rateLimiting']): Record { 15 | const table: Record = {} 16 | if (config === undefined || config.maxByClaims === undefined) { 17 | return table 18 | } 19 | 20 | for (const { claim, claimValue: value, max } of config.maxByClaims) { 21 | if (!(claim in table)) { 22 | table[claim] = { values: {} } 23 | } 24 | 25 | table[claim].values[value] = max 26 | } 27 | 28 | return table 29 | } 30 | 31 | export default fastifyPlugin(async (fastify: FastifyInstance) => { 32 | const { config } = fastify.platformatic 33 | const { rateLimiting: rateLimitingConfig } = config 34 | const maxByClaimLookupTable = buildMaxByClaimLookupTable(rateLimitingConfig) 35 | const { rateLimiting } = fastify.ai 36 | 37 | await fastify.register(fastifyRateLimit, { 38 | // Note: user can override this by setting it in their platformatic config 39 | max: async (req, key) => { 40 | if (rateLimiting.max !== undefined) { 41 | return await rateLimiting.max(req, key) 42 | } 43 | 44 | if (rateLimitingConfig !== undefined) { 45 | if ( 46 | req.user !== undefined && 47 | req.user !== null && 48 | typeof req.user === 'object' 49 | ) { 50 | for (const claim of Object.keys(req.user)) { 51 | if (claim in maxByClaimLookupTable) { 52 | const { values } = maxByClaimLookupTable[claim] 53 | 54 | // @ts-expect-error 55 | if (req.user[claim] in values) { 56 | // @ts-expect-error 57 | return values[req.user[claim]] 58 | } 59 | } 60 | } 61 | } 62 | 63 | const { max } = rateLimitingConfig 64 | if (max !== undefined) { 65 | return max 66 | } 67 | } 68 | 69 | return 1000 // default used in @fastify/rate-limit 70 | }, 71 | // Note: user can override this by setting it in their platformatic config 72 | allowList: async (req, key) => { 73 | if (rateLimiting.allowList !== undefined) { 74 | return await rateLimiting.allowList(req, key) 75 | } else if (rateLimitingConfig?.allowList !== undefined) { 76 | return rateLimitingConfig.allowList.includes(key) 77 | } 78 | return false 79 | }, 80 | onBanReach: (req, key) => { 81 | if (rateLimiting.onBanReach !== undefined) { 82 | rateLimiting.onBanReach(req, key) 83 | } 84 | }, 85 | keyGenerator: async (req) => { 86 | if (rateLimiting.keyGenerator !== undefined) { 87 | return await rateLimiting.keyGenerator(req) 88 | } else { 89 | return req.ip 90 | } 91 | }, 92 | errorResponseBuilder: (req, context) => { 93 | if (rateLimiting.errorResponseBuilder !== undefined) { 94 | return rateLimiting.errorResponseBuilder(req, context) 95 | } else { 96 | const RateLimitError = createError('RATE_LIMITED', 'Rate limit exceeded, retry in %s') 97 | const err = new RateLimitError(context.after) 98 | err.statusCode = 429 // TODO: use context.statusCode https://github.com/fastify/fastify-rate-limit/pull/366 99 | return err 100 | } 101 | }, 102 | onExceeding: (req, key) => { 103 | if (rateLimiting.onExceeded !== undefined) { 104 | rateLimiting.onExceeded(req, key) 105 | } 106 | }, 107 | onExceeded: (req, key) => { 108 | if (rateLimiting.onExceeding !== undefined) { 109 | rateLimiting.onExceeding(req, key) 110 | } 111 | }, 112 | ...rateLimitingConfig 113 | }) 114 | }) 115 | -------------------------------------------------------------------------------- /index.d.ts: -------------------------------------------------------------------------------- 1 | import { ReadableStream } from 'node:stream/web' 2 | import { PlatformaticApp } from '@platformatic/service' 3 | import { errorResponseBuilderContext } from '@fastify/rate-limit' 4 | import { AiWarpConfig } from './config.js' 5 | 6 | type ChatHistory = Array<{ 7 | prompt: string 8 | response: string 9 | }> 10 | 11 | declare module 'fastify' { 12 | interface FastifyInstance { 13 | platformatic: PlatformaticApp 14 | ai: { 15 | /** 16 | * Send a prompt to the AI provider and receive the full response. 17 | */ 18 | warp: (request: FastifyRequest, prompt: string, chatHistory?: ChatHistory) => Promise 19 | 20 | /** 21 | * Send a prompt to the AI provider and receive a streamed response. 22 | */ 23 | warpStream: (request: FastifyRequest, prompt: string, chatHistory?: ChatHistory) => Promise 24 | 25 | /** 26 | * A function to be called before warp() returns it's result. It can 27 | * modify the response and can be synchronous or asynchronous. 28 | */ 29 | preResponseCallback?: ((request: FastifyRequest, response: string) => void) | 30 | ((request: FastifyRequest, response: string) => string) | 31 | ((request: FastifyRequest, response: string) => Promise) | 32 | ((request: FastifyRequest, response: string) => Promise) 33 | 34 | /** 35 | * A function to be called on each chunk present in the `ReadableStream` 36 | * returned by warpStream(). It can modify each individual chunk and can 37 | * be synchronous or asynchronous. 38 | */ 39 | preResponseChunkCallback?: ((request: FastifyRequest, response: string) => void) | 40 | ((request: FastifyRequest, response: string) => string) | 41 | ((request: FastifyRequest, response: string) => Promise) | 42 | ((request: FastifyRequest, response: string) => Promise) 43 | 44 | rateLimiting: { 45 | /** 46 | * Callback for determining the max amount of requests a client can 47 | * send before they are rate limited. If the `rateLimiting.max` 48 | * property is defined in the Platformatic config, this method will 49 | * not be called. 50 | * @see https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options 51 | */ 52 | max?: ((request: FastifyRequest, key: string) => number) | ((request: FastifyRequest, key: string) => Promise) 53 | 54 | /** 55 | * Callback for determining the clients excluded from rate limiting. If 56 | * the `rateLimiting.allowList` property is defined in the Platformatic 57 | * config, this method will not be called. 58 | * @see https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options 59 | */ 60 | allowList?: (request: FastifyRequest, key: string) => boolean | Promise 61 | 62 | /** 63 | * Callback executed when a client reaches the ban threshold. 64 | * @see https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options 65 | */ 66 | onBanReach?: (request: FastifyRequest, key: string) => void 67 | 68 | /** 69 | * Callback for generating the unique rate limiting identifier for each client. 70 | * @see https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options 71 | */ 72 | keyGenerator?: (request: FastifyRequest) => string | number | Promise 73 | 74 | /** 75 | * Callback for generating custom response objects for rate limiting errors. 76 | * @see https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options 77 | */ 78 | errorResponseBuilder?: ( 79 | request: FastifyRequest, 80 | context: errorResponseBuilderContext 81 | ) => object 82 | 83 | /** 84 | * Callback executed before a client exceeds their request limit. 85 | * @see https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options 86 | */ 87 | onExceeding?: (request: FastifyRequest, key: string) => void 88 | 89 | /** 90 | * Callback executed after a client exceeds their request limit. 91 | * @see https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options 92 | */ 93 | onExceeded?: (request: FastifyRequest, key: string) => void 94 | } 95 | } 96 | } 97 | } 98 | 99 | export { PlatformaticApp, AiWarpConfig } 100 | -------------------------------------------------------------------------------- /static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | AI Warp 7 | 8 | 9 | 10 | 11 | 12 | 13 | 18 | 19 |
20 | Main Illustration 21 |
22 | 23 |
24 |

Welcome to

25 |

Platformatic Ai-Warp

26 |
27 | 28 |
29 |
30 |
31 |

Generate SQL schema

32 |

33 | for an ecommerce's online store 34 | 35 | 36 | 37 | 38 | 39 |

40 |
41 |
42 |

Calculate

43 |

44 | a mathematical formula 45 | 46 | 47 | 48 | 49 | 50 |

51 |
52 |
53 | 54 |
55 |
56 |

Generate a Fastify plugin

57 |

58 | that provides a route for users to sign up 59 | 60 | 61 | 62 | 63 | 64 |

65 |
66 | 67 |
68 |

Decode

69 |

70 | a base64 string 71 | 72 | 73 | 74 | 75 | 76 |

77 |
78 |
79 |
80 | 81 |
82 | 83 | 86 |
87 | 88 | 91 | 92 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /ai-providers/azure.ts: -------------------------------------------------------------------------------- 1 | import { ReadableStream, ReadableByteStreamController, UnderlyingByteSource } from 'stream/web' 2 | import { AiProvider, ChatHistory, NoContentError, StreamChunkCallback } from './provider.js' 3 | import { AiStreamEvent, encodeEvent } from './event.js' 4 | import { AzureKeyCredential, ChatCompletions, ChatRequestMessageUnion, EventStream, OpenAIClient } from '@azure/openai' 5 | 6 | type AzureStreamResponse = EventStream 7 | 8 | class AzureByteSource implements UnderlyingByteSource { 9 | type: 'bytes' = 'bytes' 10 | response: AzureStreamResponse 11 | reader?: ReadableStreamDefaultReader 12 | chunkCallback?: StreamChunkCallback 13 | 14 | constructor (response: AzureStreamResponse, chunkCallback?: StreamChunkCallback) { 15 | this.response = response 16 | this.chunkCallback = chunkCallback 17 | } 18 | 19 | start (): void { 20 | this.reader = this.response.getReader() 21 | } 22 | 23 | async pull (controller: ReadableByteStreamController): Promise { 24 | // start() defines this.reader and is called before this 25 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 26 | const { done, value } = await this.reader!.read() 27 | 28 | if (done !== undefined && done) { 29 | controller.close() 30 | return 31 | } 32 | 33 | if (value.choices.length === 0) { 34 | const error = new NoContentError('Azure OpenAI') 35 | 36 | const eventData: AiStreamEvent = { 37 | event: 'error', 38 | data: error 39 | } 40 | controller.enqueue(encodeEvent(eventData)) 41 | controller.close() 42 | 43 | return 44 | } 45 | 46 | const { delta } = value.choices[0] 47 | if (delta === undefined || delta.content === null) { 48 | const error = new NoContentError('Azure OpenAI') 49 | 50 | const eventData: AiStreamEvent = { 51 | event: 'error', 52 | data: error 53 | } 54 | controller.enqueue(encodeEvent(eventData)) 55 | controller.close() 56 | 57 | return 58 | } 59 | 60 | let response = delta.content 61 | if (this.chunkCallback !== undefined) { 62 | response = await this.chunkCallback(response) 63 | } 64 | 65 | const eventData: AiStreamEvent = { 66 | event: 'content', 67 | data: { 68 | response 69 | } 70 | } 71 | controller.enqueue(encodeEvent(eventData)) 72 | } 73 | } 74 | 75 | interface AzureProviderCtorOptions { 76 | endpoint: string 77 | apiKey: string 78 | deploymentName: string 79 | allowInsecureConnections?: boolean 80 | } 81 | 82 | export class AzureProvider implements AiProvider { 83 | deploymentName: string 84 | client: OpenAIClient 85 | 86 | constructor ({ endpoint, apiKey, deploymentName, allowInsecureConnections }: AzureProviderCtorOptions) { 87 | this.deploymentName = deploymentName 88 | 89 | this.client = new OpenAIClient( 90 | endpoint, 91 | new AzureKeyCredential(apiKey), 92 | { 93 | allowInsecureConnection: allowInsecureConnections 94 | } 95 | ) 96 | } 97 | 98 | async ask (prompt: string, chatHistory?: ChatHistory): Promise { 99 | const { choices } = await this.client.getChatCompletions(this.deploymentName, [ 100 | ...this.chatHistoryToMessages(chatHistory), 101 | { role: 'user', content: prompt } 102 | ]) 103 | 104 | if (choices.length === 0) { 105 | throw new NoContentError('Azure OpenAI') 106 | } 107 | 108 | const { message } = choices[0] 109 | if (message === undefined || message.content === null) { 110 | throw new NoContentError('Azure OpenAI') 111 | } 112 | 113 | return message.content 114 | } 115 | 116 | async askStream (prompt: string, chunkCallback?: StreamChunkCallback, chatHistory?: ChatHistory): Promise { 117 | const response = await this.client.streamChatCompletions(this.deploymentName, [ 118 | ...this.chatHistoryToMessages(chatHistory), 119 | { role: 'user', content: prompt } 120 | ]) 121 | 122 | return new ReadableStream(new AzureByteSource(response, chunkCallback)) 123 | } 124 | 125 | private chatHistoryToMessages (chatHistory?: ChatHistory): ChatRequestMessageUnion[] { 126 | if (chatHistory === undefined) { 127 | return [] 128 | } 129 | 130 | const messages: ChatRequestMessageUnion[] = [] 131 | for (const previousInteraction of chatHistory) { 132 | messages.push({ role: 'user', content: previousInteraction.prompt }) 133 | messages.push({ role: 'assistant', content: previousInteraction.response }) 134 | } 135 | 136 | return messages 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /ai-providers/open-ai.ts: -------------------------------------------------------------------------------- 1 | import { ReadableStream, UnderlyingByteSource, ReadableByteStreamController } from 'node:stream/web' 2 | import OpenAI from 'openai' 3 | import { AiProvider, ChatHistory, NoContentError, StreamChunkCallback } from './provider.js' 4 | import { ReadableStream as ReadableStreamPolyfill } from 'web-streams-polyfill' 5 | import { fetch } from 'undici' 6 | import { ChatCompletionChunk, ChatCompletionMessageParam } from 'openai/resources/index' 7 | import { AiStreamEvent, encodeEvent } from './event.js' 8 | import createError from '@fastify/error' 9 | 10 | const InvalidTypeError = createError('DESERIALIZING_ERROR', 'Deserializing error: %s', 500) 11 | 12 | class OpenAiByteSource implements UnderlyingByteSource { 13 | type: 'bytes' = 'bytes' 14 | polyfillStream: ReadableStreamPolyfill 15 | reader?: ReadableStreamDefaultReader 16 | chunkCallback?: StreamChunkCallback 17 | 18 | constructor (polyfillStream: ReadableStreamPolyfill, chunkCallback?: StreamChunkCallback) { 19 | this.polyfillStream = polyfillStream 20 | this.chunkCallback = chunkCallback 21 | } 22 | 23 | start (): void { 24 | this.reader = this.polyfillStream.getReader() 25 | } 26 | 27 | async pull (controller: ReadableByteStreamController): Promise { 28 | // start() defines this.reader and is called before this 29 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 30 | const { done, value } = await this.reader!.read() 31 | 32 | if (done !== undefined && done) { 33 | controller.close() 34 | return 35 | } 36 | 37 | if (!(value instanceof Uint8Array)) { 38 | // This really shouldn't happen but just in case + typescript likes 39 | const error = new InvalidTypeError('OpenAI stream value not a Uint8Array') 40 | 41 | const eventData: AiStreamEvent = { 42 | event: 'error', 43 | data: error 44 | } 45 | controller.enqueue(encodeEvent(eventData)) 46 | controller.close() 47 | 48 | return 49 | } 50 | 51 | const jsonString = Buffer.from(value).toString('utf8') 52 | const chunk: ChatCompletionChunk = JSON.parse(jsonString) 53 | 54 | if (chunk.choices.length === 0) { 55 | const error = new NoContentError('OpenAI stream') 56 | 57 | const eventData: AiStreamEvent = { 58 | event: 'error', 59 | data: error 60 | } 61 | controller.enqueue(encodeEvent(eventData)) 62 | controller.close() 63 | 64 | return 65 | } 66 | 67 | const { content } = chunk.choices[0].delta 68 | 69 | let response = content ?? '' 70 | if (this.chunkCallback !== undefined) { 71 | response = await this.chunkCallback(response) 72 | } 73 | 74 | const eventData: AiStreamEvent = { 75 | event: 'content', 76 | data: { 77 | response 78 | } 79 | } 80 | controller.enqueue(encodeEvent(eventData)) 81 | } 82 | } 83 | 84 | interface OpenAiProviderCtorOptions { 85 | model: string 86 | apiKey: string 87 | } 88 | 89 | export class OpenAiProvider implements AiProvider { 90 | model: string 91 | client: OpenAI 92 | 93 | constructor ({ model, apiKey }: OpenAiProviderCtorOptions) { 94 | this.model = model 95 | // @ts-expect-error 96 | this.client = new OpenAI({ apiKey, fetch }) 97 | } 98 | 99 | async ask (prompt: string, chatHistory?: ChatHistory): Promise { 100 | const response = await this.client.chat.completions.create({ 101 | model: this.model, 102 | messages: [ 103 | ...this.chatHistoryToMessages(chatHistory), 104 | { role: 'user', content: prompt } 105 | ], 106 | stream: false 107 | }) 108 | 109 | if (response.choices.length === 0) { 110 | throw new NoContentError('OpenAI') 111 | } 112 | 113 | const { content } = response.choices[0].message 114 | if (content === null) { 115 | throw new NoContentError('OpenAI') 116 | } 117 | 118 | return content 119 | } 120 | 121 | async askStream (prompt: string, chunkCallback?: StreamChunkCallback, chatHistory?: ChatHistory): Promise { 122 | const response = await this.client.chat.completions.create({ 123 | model: this.model, 124 | messages: [ 125 | ...this.chatHistoryToMessages(chatHistory), 126 | { role: 'user', content: prompt } 127 | ], 128 | stream: true 129 | }) 130 | return new ReadableStream(new OpenAiByteSource(response.toReadableStream() as ReadableStreamPolyfill, chunkCallback)) 131 | } 132 | 133 | private chatHistoryToMessages (chatHistory?: ChatHistory): ChatCompletionMessageParam[] { 134 | if (chatHistory === undefined) { 135 | return [] 136 | } 137 | 138 | const messages: ChatCompletionMessageParam[] = [] 139 | for (const previousInteraction of chatHistory) { 140 | messages.push({ role: 'user', content: previousInteraction.prompt }) 141 | messages.push({ role: 'assistant', content: previousInteraction.response }) 142 | } 143 | 144 | return messages 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /tests/unit/generator.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-floating-promises */ 2 | import { describe, it, afterEach } from 'node:test' 3 | import assert from 'node:assert' 4 | import { existsSync } from 'node:fs' 5 | import { mkdir, rm, readFile } from 'node:fs/promises' 6 | import { join } from 'node:path' 7 | import AiWarpGenerator from '../../lib/generator.js' 8 | import { generateGlobalTypesFile } from '../../lib/templates/types.js' 9 | import { generatePluginWithTypesSupport } from '@platformatic/generators/lib/create-plugin.js' 10 | import { mockAllProviders } from '../utils/mocks/index.js' 11 | mockAllProviders() 12 | 13 | const tempDirBase = join(import.meta.dirname, 'tmp') 14 | 15 | let counter = 0 16 | export async function getTempDir (baseDir: string): Promise { 17 | if (baseDir === undefined) { 18 | baseDir = import.meta.dirname 19 | } 20 | const dir = join(baseDir, `platformatic-generators-${process.pid}-${Date.now()}-${counter++}`) 21 | try { 22 | await mkdir(dir, { recursive: true }) 23 | } catch (err) { 24 | // do nothing 25 | } 26 | return dir 27 | } 28 | 29 | describe('AiWarpGenerator', () => { 30 | afterEach(async () => { 31 | try { 32 | await rm(tempDirBase, { recursive: true }) 33 | } catch (err) { 34 | // do nothing 35 | } 36 | }) 37 | 38 | it('generates global.d.ts correctly', async () => { 39 | const dir = await getTempDir(tempDirBase) 40 | 41 | const generator = new AiWarpGenerator() 42 | generator.setConfig({ 43 | targetDirectory: dir, 44 | aiWarpPackageJsonPath: join(import.meta.dirname, '..', '..', 'package.json'), 45 | aiProvider: 'openai', 46 | aiModel: 'gpt-3.5-turbo' 47 | }) 48 | await generator.run() 49 | 50 | const globalTypes = await readFile(join(dir, 'global.d.ts'), 'utf8') 51 | assert.strictEqual(globalTypes, generateGlobalTypesFile('@platformatic/ai-warp')) 52 | }) 53 | 54 | it('adds env variables to .env', async () => { 55 | const dir = await getTempDir(tempDirBase) 56 | 57 | const generator = new AiWarpGenerator() 58 | generator.setConfig({ 59 | targetDirectory: dir, 60 | aiWarpPackageJsonPath: join(import.meta.dirname, '..', '..', 'package.json'), 61 | aiProvider: 'openai', 62 | aiModel: 'gpt-3.5-turbo' 63 | }) 64 | await generator.run() 65 | 66 | // Env file has the api key fields for all providers 67 | const envFile = await readFile(join(dir, '.env'), 'utf8') 68 | assert.ok(envFile.includes('PLT_AI_API_KEY')) 69 | 70 | const sampleEnvFile = await readFile(join(dir, '.env.sample'), 'utf8') 71 | assert.ok(sampleEnvFile.includes('PLT_AI_API_KEY')) 72 | }) 73 | 74 | it('generates platformatic.json correctly', async () => { 75 | const dir = await getTempDir(tempDirBase) 76 | 77 | const generator = new AiWarpGenerator() 78 | generator.setConfig({ 79 | targetDirectory: dir, 80 | aiWarpPackageJsonPath: join(import.meta.dirname, '..', '..', 'package.json'), 81 | aiProvider: 'openai', 82 | aiModel: 'gpt-3.5-turbo' 83 | }) 84 | await generator.run() 85 | 86 | let configFile = JSON.parse(await readFile(join(dir, 'platformatic.json'), 'utf8')) 87 | assert.deepStrictEqual(configFile.aiProvider, { 88 | openai: { 89 | model: 'gpt-3.5-turbo', 90 | apiKey: '{PLT_AI_API_KEY}' 91 | } 92 | }) 93 | 94 | generator.setConfig({ 95 | aiProvider: 'mistral', 96 | aiModel: 'open-mistral-7b' 97 | }) 98 | await generator.run() 99 | 100 | configFile = JSON.parse(await readFile(join(dir, 'platformatic.json'), 'utf8')) 101 | assert.deepStrictEqual(configFile.aiProvider, { 102 | mistral: { 103 | model: 'open-mistral-7b', 104 | apiKey: '{PLT_AI_API_KEY}' 105 | } 106 | }) 107 | }) 108 | 109 | it('doesn\'t generate a plugin when not wanted', async () => { 110 | const dir = await getTempDir(tempDirBase) 111 | 112 | const generator = new AiWarpGenerator() 113 | generator.setConfig({ 114 | targetDirectory: dir, 115 | aiWarpPackageJsonPath: join(import.meta.dirname, '..', '..', 'package.json'), 116 | aiProvider: 'openai', 117 | aiModel: 'gpt-3.5-turbo' 118 | }) 119 | await generator.run() 120 | 121 | const pluginsDirectory = join(dir, 'plugins') 122 | assert.strictEqual(existsSync(pluginsDirectory), false) 123 | }) 124 | 125 | it('generates expected js example plugin', async () => { 126 | const dir = await getTempDir(tempDirBase) 127 | 128 | const generator = new AiWarpGenerator() 129 | generator.setConfig({ 130 | targetDirectory: dir, 131 | aiWarpPackageJsonPath: join(import.meta.dirname, '..', '..', 'package.json'), 132 | aiProvider: 'openai', 133 | aiModel: 'gpt-3.5-turbo', 134 | plugin: true 135 | }) 136 | await generator.run() 137 | 138 | const pluginsDirectory = join(dir, 'plugins') 139 | assert.strictEqual(existsSync(pluginsDirectory), true) 140 | 141 | const exampleJsPlugin = await readFile(join(pluginsDirectory, 'example.js'), 'utf8') 142 | assert.strictEqual(exampleJsPlugin, generatePluginWithTypesSupport(false).contents) 143 | }) 144 | 145 | it('generates expected ts example plugin', async () => { 146 | const dir = await getTempDir(tempDirBase) 147 | 148 | const generator = new AiWarpGenerator() 149 | generator.setConfig({ 150 | targetDirectory: dir, 151 | aiWarpPackageJsonPath: join(import.meta.dirname, '..', '..', 'package.json'), 152 | aiProvider: 'openai', 153 | aiModel: 'gpt-3.5-turbo', 154 | plugin: true, 155 | typescript: true 156 | }) 157 | await generator.run() 158 | 159 | const pluginsDirectory = join(dir, 'plugins') 160 | assert.strictEqual(existsSync(pluginsDirectory), true) 161 | 162 | const exampleTsPlugin = await readFile(join(pluginsDirectory, 'example.ts'), 'utf8') 163 | assert.strictEqual(exampleTsPlugin, generatePluginWithTypesSupport(true).contents) 164 | }) 165 | }) 166 | -------------------------------------------------------------------------------- /ai-providers/llama2.ts: -------------------------------------------------------------------------------- 1 | import { ReadableByteStreamController, ReadableStream, UnderlyingByteSource } from 'stream/web' 2 | import { FastifyLoggerInstance } from 'fastify' 3 | import { 4 | LLamaChatPromptOptions, 5 | LlamaChatSession, 6 | LlamaContext, 7 | LlamaModel 8 | } from 'node-llama-cpp' 9 | import { AiProvider, ChatHistory, StreamChunkCallback } from './provider.js' 10 | import { AiStreamEvent, encodeEvent } from './event.js' 11 | 12 | interface ChunkQueueNode { 13 | chunk: number[] 14 | next?: ChunkQueueNode 15 | } 16 | 17 | class ChunkQueue { 18 | private size: number = 0 19 | private head?: ChunkQueueNode 20 | private tail?: ChunkQueueNode 21 | 22 | getSize (): number { 23 | return this.size 24 | } 25 | 26 | push (chunk: number[]): void { 27 | this.size++ 28 | 29 | const node: ChunkQueueNode = { chunk } 30 | if (this.head === undefined || this.tail === undefined) { 31 | this.head = node 32 | this.tail = node 33 | } else { 34 | this.tail.next = node 35 | this.tail = node 36 | } 37 | } 38 | 39 | pop (): number[] | undefined { 40 | if (this.head === undefined) { 41 | return undefined 42 | } 43 | 44 | this.size-- 45 | 46 | const chunk = this.head.chunk 47 | this.head = this.head.next 48 | 49 | if (this.size === 0) { 50 | this.tail = undefined 51 | } 52 | 53 | return chunk 54 | } 55 | } 56 | 57 | class Llama2ByteSource implements UnderlyingByteSource { 58 | type: 'bytes' = 'bytes' 59 | session: LlamaChatSession 60 | chunkCallback?: StreamChunkCallback 61 | backloggedChunks: ChunkQueue = new ChunkQueue() 62 | finished: boolean = false 63 | controller?: ReadableByteStreamController 64 | abortController: AbortController 65 | 66 | constructor (session: LlamaChatSession, prompt: string, logger: FastifyLoggerInstance, chunkCallback?: StreamChunkCallback) { 67 | this.session = session 68 | this.chunkCallback = chunkCallback 69 | this.abortController = new AbortController() 70 | 71 | session.prompt(prompt, { 72 | onToken: this.onToken, 73 | signal: this.abortController.signal 74 | }).then(() => { 75 | this.finished = true 76 | // Don't close the stream if we still have chunks to send 77 | if (this.backloggedChunks.getSize() === 0 && this.controller !== undefined) { 78 | this.controller.close() 79 | } 80 | }).catch((err: any) => { 81 | this.finished = true 82 | logger.info({ err }) 83 | if (!this.abortController.signal.aborted && this.controller !== undefined) { 84 | try { 85 | this.controller.close() 86 | } catch (err) { 87 | logger.info({ err }) 88 | } 89 | } 90 | }) 91 | } 92 | 93 | cancel (): void { 94 | this.abortController.abort() 95 | } 96 | 97 | onToken: LLamaChatPromptOptions['onToken'] = async (chunk) => { 98 | if (this.controller === undefined) { 99 | // Stream hasn't started yet, added it to the backlog queue 100 | this.backloggedChunks.push(chunk) 101 | return 102 | } 103 | 104 | try { 105 | await this.clearBacklog() 106 | await this.enqueueChunk(chunk) 107 | // Ignore all errors, we can't do anything about them 108 | // TODO: Log these errors 109 | } catch (err) { 110 | console.error(err) 111 | } 112 | } 113 | 114 | private async enqueueChunk (chunk: number[]): Promise { 115 | if (this.controller === undefined) { 116 | throw new Error('tried enqueueing chunk before stream started') 117 | } 118 | 119 | let response = this.session.context.decode(chunk) 120 | if (this.chunkCallback !== undefined) { 121 | response = await this.chunkCallback(response) 122 | } 123 | 124 | if (response === '') { 125 | response = '\n' // It seems empty chunks are newlines 126 | } 127 | 128 | const eventData: AiStreamEvent = { 129 | event: 'content', 130 | data: { 131 | response 132 | } 133 | } 134 | this.controller.enqueue(encodeEvent(eventData)) 135 | 136 | if (this.backloggedChunks.getSize() === 0 && this.finished) { 137 | this.controller.close() 138 | } 139 | } 140 | 141 | async clearBacklog (): Promise { 142 | if (this.backloggedChunks.getSize() === 0) { 143 | return 144 | } 145 | 146 | let backloggedChunk = this.backloggedChunks.pop() 147 | while (backloggedChunk !== undefined) { 148 | // Each chunk needs to be sent in order, can't run all of these at once 149 | await this.enqueueChunk(backloggedChunk) 150 | backloggedChunk = this.backloggedChunks.pop() 151 | } 152 | } 153 | 154 | start (controller: ReadableByteStreamController): void { 155 | this.controller = controller 156 | this.clearBacklog().catch(err => { 157 | throw err 158 | }) 159 | } 160 | } 161 | 162 | interface Llama2ProviderCtorOptions { 163 | modelPath: string 164 | logger: FastifyLoggerInstance 165 | } 166 | 167 | export class Llama2Provider implements AiProvider { 168 | model: LlamaModel 169 | logger: FastifyLoggerInstance 170 | 171 | constructor ({ modelPath, logger }: Llama2ProviderCtorOptions) { 172 | this.model = new LlamaModel({ modelPath }) 173 | this.logger = logger 174 | } 175 | 176 | async ask (prompt: string, chatHistory?: ChatHistory): Promise { 177 | const context = new LlamaContext({ model: this.model }) 178 | const session = new LlamaChatSession({ 179 | context, 180 | conversationHistory: chatHistory 181 | }) 182 | const response = await session.prompt(prompt) 183 | 184 | return response 185 | } 186 | 187 | async askStream (prompt: string, chunkCallback?: StreamChunkCallback, chatHistory?: ChatHistory): Promise { 188 | const context = new LlamaContext({ model: this.model }) 189 | const session = new LlamaChatSession({ 190 | context, 191 | conversationHistory: chatHistory 192 | }) 193 | 194 | return new ReadableStream(new Llama2ByteSource(session, prompt, this.logger, chunkCallback)) 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /tests/e2e/rate-limiting.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-floating-promises */ 2 | import { it } from 'node:test' 3 | import assert from 'node:assert' 4 | import fastifyPlugin from 'fastify-plugin' 5 | import { AiWarpConfig } from '../../config.js' 6 | import { buildAiWarpApp } from '../utils/stackable.js' 7 | import { authConfig, createToken } from '../utils/auth.js' 8 | import { mockAllProviders } from '../utils/mocks/index.js' 9 | mockAllProviders() 10 | 11 | const aiProvider: AiWarpConfig['aiProvider'] = { 12 | openai: { 13 | model: 'gpt-3.5-turbo', 14 | apiKey: '' 15 | } 16 | } 17 | 18 | it('calls ai.rateLimiting.max callback', async () => { 19 | const [app, port] = await buildAiWarpApp({ aiProvider }) 20 | 21 | try { 22 | const expectedMax = 100 23 | let callbackCalled = false 24 | await app.register(fastifyPlugin(async () => { 25 | app.ai.rateLimiting.max = () => { 26 | callbackCalled = true 27 | return expectedMax 28 | } 29 | })) 30 | 31 | await app.start() 32 | 33 | const res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 34 | method: 'POST', 35 | headers: { 36 | 'content-type': 'application/json' 37 | }, 38 | body: JSON.stringify({ 39 | prompt: 'asd' 40 | }) 41 | }) 42 | assert.strictEqual(callbackCalled, true) 43 | assert.strictEqual(res.headers.get('x-ratelimit-limit'), `${expectedMax}`) 44 | } finally { 45 | await app.close() 46 | } 47 | }) 48 | 49 | it('calls ai.rateLimiting.allowList callback', async () => { 50 | const [app, port] = await buildAiWarpApp({ aiProvider }) 51 | 52 | try { 53 | let callbackCalled = false 54 | app.register(fastifyPlugin(async () => { 55 | app.ai.rateLimiting.allowList = () => { 56 | callbackCalled = true 57 | return true 58 | } 59 | })) 60 | 61 | await app.start() 62 | 63 | await fetch(`http://localhost:${port}/api/v1/prompt`, { 64 | method: 'POST', 65 | headers: { 66 | 'content-type': 'application/json' 67 | }, 68 | body: JSON.stringify({ 69 | prompt: 'asd' 70 | }) 71 | }) 72 | assert.strictEqual(callbackCalled, true) 73 | } finally { 74 | await app.close() 75 | } 76 | }) 77 | 78 | it('calls ai.rateLimiting.onBanReach callback', async () => { 79 | const [app, port] = await buildAiWarpApp({ 80 | aiProvider, 81 | rateLimiting: { 82 | max: 0, 83 | ban: 0 84 | } 85 | }) 86 | 87 | try { 88 | let onBanReachCalled = false 89 | let errorResponseBuilderCalled = false 90 | app.register(fastifyPlugin(async () => { 91 | app.ai.rateLimiting.onBanReach = () => { 92 | onBanReachCalled = true 93 | } 94 | 95 | app.ai.rateLimiting.errorResponseBuilder = () => { 96 | errorResponseBuilderCalled = true 97 | return { message: 'rate limited' } 98 | } 99 | })) 100 | 101 | await app.start() 102 | 103 | await fetch(`http://localhost:${port}/api/v1/prompt`, { 104 | method: 'POST', 105 | headers: { 106 | 'content-type': 'application/json' 107 | }, 108 | body: JSON.stringify({ 109 | prompt: 'asd' 110 | }) 111 | }) 112 | assert.strictEqual(onBanReachCalled, true) 113 | assert.strictEqual(errorResponseBuilderCalled, true) 114 | } finally { 115 | await app.close() 116 | } 117 | }) 118 | 119 | it('calls ai.rateLimiting.keyGenerator callback', async () => { 120 | const [app, port] = await buildAiWarpApp({ aiProvider }) 121 | 122 | try { 123 | let callbackCalled = false 124 | app.register(fastifyPlugin(async () => { 125 | app.ai.rateLimiting.keyGenerator = (req) => { 126 | callbackCalled = true 127 | return req.ip 128 | } 129 | })) 130 | 131 | await app.start() 132 | 133 | await fetch(`http://localhost:${port}/api/v1/prompt`, { 134 | method: 'POST', 135 | headers: { 136 | 'content-type': 'application/json' 137 | }, 138 | body: JSON.stringify({ 139 | prompt: 'asd' 140 | }) 141 | }) 142 | assert.strictEqual(callbackCalled, true) 143 | } finally { 144 | await app.close() 145 | } 146 | }) 147 | 148 | it('calls ai.rateLimiting.errorResponseBuilder callback', async () => { 149 | const [app, port] = await buildAiWarpApp({ aiProvider }) 150 | 151 | try { 152 | let callbackCalled = false 153 | app.register(fastifyPlugin(async () => { 154 | app.ai.rateLimiting.max = () => 0 155 | app.ai.rateLimiting.errorResponseBuilder = () => { 156 | callbackCalled = true 157 | return { message: 'rate limited' } 158 | } 159 | })) 160 | 161 | await app.start() 162 | 163 | await fetch(`http://localhost:${port}/api/v1/prompt`, { 164 | method: 'POST', 165 | headers: { 166 | 'content-type': 'application/json' 167 | }, 168 | body: JSON.stringify({ 169 | prompt: 'asd' 170 | }) 171 | }) 172 | assert.strictEqual(callbackCalled, true) 173 | } finally { 174 | await app.close() 175 | } 176 | }) 177 | 178 | it('uses the max for a specific claim', async () => { 179 | const [app, port] = await buildAiWarpApp({ 180 | aiProvider, 181 | rateLimiting: { 182 | maxByClaims: [ 183 | { 184 | claim: 'rateLimitMax', 185 | claimValue: '10', 186 | max: 10 187 | }, 188 | { 189 | claim: 'rateLimitMax', 190 | claimValue: '100', 191 | max: 100 192 | } 193 | ] 194 | }, 195 | auth: authConfig 196 | }) 197 | 198 | try { 199 | await app.start() 200 | 201 | let res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 202 | method: 'POST', 203 | headers: { 204 | Authorization: `Bearer ${createToken({ rateLimitMax: '10' })}`, 205 | 'content-type': 'application/json' 206 | }, 207 | body: JSON.stringify({ 208 | prompt: 'asd' 209 | }) 210 | }) 211 | assert.strictEqual(res.headers.get('x-ratelimit-limit'), '10') 212 | 213 | res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 214 | method: 'POST', 215 | headers: { 216 | Authorization: `Bearer ${createToken({ rateLimitMax: '100' })}`, 217 | 'content-type': 'application/json' 218 | }, 219 | body: JSON.stringify({ 220 | prompt: 'asd' 221 | }) 222 | }) 223 | assert.strictEqual(res.headers.get('x-ratelimit-limit'), '100') 224 | } finally { 225 | await app.close() 226 | } 227 | }) 228 | -------------------------------------------------------------------------------- /docs/plugin-api.md: -------------------------------------------------------------------------------- 1 | # AI Warp API 2 | 3 | Documentation on the methods and properties availabile to plugins added onto this stackable. 4 | 5 | All of these exist under the `fastify.ai` object. 6 | 7 | ## `fastify.ai.warp()` 8 | 9 | Send a prompt to the AI provider and receive the full response. 10 | 11 | Takes in: 12 | 13 | * `request` (`FastifyRequest`) The request object 14 | * `prompt` (`string`) The prompt to send to the AI provider 15 | 16 | Returns: 17 | 18 | * `string` - Full response from the AI provider 19 | 20 |
21 | Example usage 22 | 23 | ```typescript 24 | const response: string = await fastify.ai.warp(request, "What's 1+1?") 25 | fastify.log.info(`response: ${response}`) 26 | ``` 27 |
28 | 29 | ## `fastify.ai.warpStream` 30 | 31 | Send a prompt to the AI provider and receive a streamed response. See [here](./rest-api.md#post-apiv1stream) for more information on the contents of the stream. 32 | 33 | Takes in: 34 | 35 | * `request` (`FastifyRequest`) The request object 36 | * `prompt` (`string`) The prompt to send to the AI provider 37 | 38 | Returns: 39 | 40 | * `ReadableStream` - Streamed response chunks from the AI provider 41 | 42 |
43 | Example usage 44 | 45 | ```typescript 46 | const response: ReadableStream = await fastify.ai.warpStream(request, "What's 1+1?") 47 | 48 | const decoder = new TextDecoder() 49 | const reader = stream.getReader() 50 | while (true) { 51 | const { done, value } = await reader.read() 52 | if (done !== undefined && done) { 53 | break 54 | } 55 | 56 | fastify.log.info(`response chunk: ${decoder.decode(value)}`) 57 | } 58 | ``` 59 |
60 | 61 | ## `fastify.ai.preResponseCallback` 62 | 63 | A function to be called before [fastify.ai.warp](#fastifyaiwarp) returns it's result. It can modify the response and can be synchronous or asynchronous. 64 | 65 |
66 | Example usage 67 | 68 | ```typescript 69 | export default fastifyPlugin(async (fastify) => { 70 | // This prefixes each response with `The AI has spoken: ` 71 | fastify.ai.preResponseCallback = (request, response) => { 72 | return `The AI has spoken: ${response}` 73 | } 74 | }) 75 | ``` 76 |
77 | 78 | ## `fastify.ai.preResponseChunkCallback` 79 | 80 | A function to be called on each chunk present in the `ReadableStream` returned by [fastify.ai.warpStream](#fastifyaiwarpstream). It can modify each individual chunk and can be synchronous or asynchronous. 81 | 82 |
83 | Example usage 84 | 85 | ```typescript 86 | export default fastifyPlugin(async (fastify) => { 87 | // This prefixes each chunk with `The AI has partially spoken: ` 88 | fastify.ai.preResponseChunkCallback = (request, response) => { 89 | return `The AI has partially spoken: ${response}` 90 | } 91 | }) 92 | ``` 93 |
94 | 95 | ## `fastify.ai.rateLimiting.max` 96 | 97 | Callback for determining the max amount of requests a client can send before they are rate limited. If the `rateLimiting.max` property is defined in the Platformatic config, this method will not be called. 98 | 99 | See [@fastify/rate-limit](https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options) options for more info. 100 | 101 |
102 | Example usage 103 | 104 | ```typescript 105 | export default fastifyPlugin(async (fastify) => { 106 | // This uses a hypothetical isUserPremium function to decide if a client gets 107 | // a request limit of 2000 or 1000. 108 | fastify.ai.rateLimiting.max = (request, key) => { 109 | return isUserPremium(request) ? 2000 : 1000 110 | } 111 | }) 112 | ``` 113 |
114 | 115 | ## `fastify.ai.rateLimiting.allowList` 116 | 117 | Callback for determining the clients excluded from rate limiting. If the `rateLimiting.allowList` property is defined in the Platformatic config, this method will not be called. 118 | 119 | See [@fastify/rate-limit](https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options) options for more info. 120 | 121 |
122 | Example usage 123 | 124 | ```typescript 125 | export default fastifyPlugin(async (fastify) => { 126 | // This allows localhost clients to be exempt from rate limiting 127 | fastify.ai.rateLimiting.allowList = (request, key) => { 128 | return key === '127.0.0.1' 129 | } 130 | }) 131 | ``` 132 |
133 | 134 | ## `fastify.ai.rateLimiting.onBanReach` 135 | 136 | Callback executed when a client reaches the ban threshold. 137 | 138 | See [@fastify/rate-limit](https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options) options for more info. 139 | 140 |
141 | Example usage 142 | 143 | ```typescript 144 | export default fastifyPlugin(async (fastify) => { 145 | // Logs when a client is rate limit banned 146 | fastify.ai.rateLimiting.onBanReach = (request, key) => { 147 | fastify.log.warn(`client ${key} has been banned for exceeding the rate limit!`) 148 | } 149 | }) 150 | ``` 151 |
152 | 153 | ## `fastify.ai.rateLimiting.keyGenerator` 154 | 155 | Callback for generating the unique rate limiting identifier for each client. 156 | 157 | See [@fastify/rate-limit](https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options) options for more info. 158 | 159 |
160 | Example usage 161 | 162 | ```typescript 163 | export default fastifyPlugin(async (fastify) => { 164 | // Uses the client's ip as the rate limit key 165 | fastify.ai.rateLimiting.keyGenerator = (request) => { 166 | return request.ip 167 | } 168 | }) 169 | ``` 170 |
171 | 172 | ## `fastify.ai.rateLimiting.errorResponseBuilder` 173 | 174 | Callback for generating custom response objects for rate limiting errors. 175 | 176 | See [@fastify/rate-limit](https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options) options for more info. 177 | 178 |
179 | Example usage 180 | 181 | ```typescript 182 | export default fastifyPlugin(async (fastify) => { 183 | fastify.ai.rateLimiting.errorResponseBuilder = (request, context) => { 184 | return { 185 | statusCode: 429, 186 | error: 'Too many requests', 187 | message: `Rate limit exceeded! Try again in ${context.after}` 188 | } 189 | } 190 | }) 191 | ``` 192 |
193 | 194 | ## `fastify.ai.rateLimiting.onExceeding` 195 | 196 | Callback executed before a client exceeds their request limit. 197 | 198 | See [@fastify/rate-limit](https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options) options for more info. 199 | 200 |
201 | Example usage 202 | 203 | ```typescript 204 | export default fastifyPlugin(async (fastify) => { 205 | fastify.ai.rateLimiting.onExceeding = (request, key) => { 206 | fastify.log.warn(`client ${key} is about to hit the request limit!`) 207 | } 208 | }) 209 | ``` 210 |
211 | 212 | ## `fastify.ai.rateLimiting.onExceeded` 213 | 214 | Callback executed after a client exceeds their request limit. 215 | 216 | See [@fastify/rate-limit](https://github.com/fastify/fastify-rate-limit?tab=readme-ov-file#options) options for more info. 217 | 218 |
219 | Example usage 220 | 221 | ```typescript 222 | export default fastifyPlugin(async (fastify) => { 223 | fastify.ai.rateLimiting.onExceeded = (request, response) => { 224 | fastify.log.warn(`client ${key} has hit the request limit!`) 225 | } 226 | }) 227 | ``` 228 |
229 | -------------------------------------------------------------------------------- /lib/generator.ts: -------------------------------------------------------------------------------- 1 | import { join, dirname } from 'node:path' 2 | import { readFile } from 'node:fs/promises' 3 | import { Generator as ServiceGenerator } from '@platformatic/service' 4 | import { BaseGenerator } from '@platformatic/generators' 5 | import { schema } from './schema.js' 6 | import { generateGlobalTypesFile } from './templates/types.js' 7 | import { generatePlugins } from '@platformatic/generators/lib/create-plugin.js' 8 | import { fileURLToPath } from 'node:url' 9 | 10 | interface PackageJson { 11 | name: string 12 | version: string 13 | devDependencies: Record 14 | } 15 | 16 | class AiWarpGenerator extends ServiceGenerator { 17 | private _packageJson: PackageJson | null = null 18 | 19 | getDefaultConfig (): { [x: string]: BaseGenerator.JSONValue } { 20 | const defaultBaseConfig = super.getDefaultConfig() 21 | // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions 22 | const dir = import.meta.dirname || dirname(fileURLToPath(import.meta.url)) 23 | const defaultConfig = { 24 | aiProvider: 'openai', 25 | aiModel: 'gpt-3.5-turbo', 26 | localSchema: false, 27 | plugin: false, 28 | tests: false, 29 | // TODO: temporary fix, when running the typescript files directly 30 | // (in tests) this goes a directory above the actual project. Exposing 31 | // temporarily until I come up with something better 32 | aiWarpPackageJsonPath: join(dir, '..', '..', 'package.json') 33 | } 34 | return Object.assign({}, defaultBaseConfig, defaultConfig) 35 | } 36 | 37 | getConfigFieldsDefinitions (): BaseGenerator.ConfigFieldDefinition[] { 38 | const serviceConfigFieldsDefs = super.getConfigFieldsDefinitions() 39 | return [ 40 | ...serviceConfigFieldsDefs, 41 | { 42 | var: 'PLT_AI_PROVIDER', 43 | label: 'What AI provider would you like to use? (e.g. openai, mistral)', 44 | default: 'openai', 45 | type: 'string', 46 | configValue: 'aiProvider' 47 | }, 48 | { 49 | // TODO: is it possible to show a list of all of the models supported here? 50 | var: 'PLT_AI_MODEL', 51 | label: 'What AI model would you like to use?', 52 | default: 'gpt-3.5-turbo', 53 | type: 'string', 54 | configValue: 'aiModel' 55 | }, 56 | { 57 | var: 'PLT_AI_API_KEY', 58 | label: 'What is your OpenAI/Mistral/Azure API key?', 59 | default: 'sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 60 | type: 'string' 61 | } 62 | ] 63 | } 64 | 65 | async _getConfigFileContents (): Promise<{ [x: string]: BaseGenerator.JSONValue }> { 66 | const baseConfig = await super._getConfigFileContents() 67 | const packageJson = await this.getStackablePackageJson() 68 | const config = { 69 | $schema: this.config.localSchema as boolean ? './stackable.schema.json' : `https://schemas.platformatic.dev/@platformatic/ai-warp/${packageJson.version}.json`, 70 | module: packageJson.name, 71 | aiProvider: {}, 72 | promptDecorators: { 73 | prefix: 'You are an AI for Acme Corp. here to answer questions anyone has.\nThe question for you to answer is: ', 74 | suffix: 'Please respond as consisely as possible.' 75 | } 76 | } 77 | switch (this.config.aiProvider) { 78 | case 'mistral': 79 | config.aiProvider = { 80 | mistral: { 81 | model: this.config.aiModel, 82 | apiKey: `{${this.getEnvVarName('PLT_AI_API_KEY')}}` 83 | } 84 | } 85 | break 86 | case 'openai': 87 | config.aiProvider = { 88 | openai: { 89 | model: this.config.aiModel, 90 | apiKey: `{${this.getEnvVarName('PLT_AI_API_KEY')}}` 91 | } 92 | } 93 | break 94 | case 'ollama': 95 | config.aiProvider = { 96 | ollama: { 97 | host: 'http://127.0.0.1:11434', 98 | model: this.config.aiModel 99 | } 100 | } 101 | break 102 | case 'azure': 103 | config.aiProvider = { 104 | azure: { 105 | endpoint: 'https://myaccount.openai.azure.com/', 106 | apiKey: `{${this.getEnvVarName('PLT_AI_API_KEY')}}`, 107 | deploymentName: this.config.aiModel 108 | } 109 | } 110 | break 111 | case 'llama2': 112 | config.aiProvider = { 113 | llama2: { 114 | modelPath: `{${this.getEnvVarName('PLT_AI_MODEL')}}` 115 | } 116 | } 117 | break 118 | default: 119 | config.aiProvider = { 120 | openai: { 121 | model: this.config.aiModel, 122 | apiKey: `{${this.getEnvVarName('PLT_AI_API_KEY')}}` 123 | } 124 | } 125 | } 126 | 127 | return Object.assign({}, baseConfig, config) 128 | } 129 | 130 | async _beforePrepare (): Promise { 131 | await super._beforePrepare() 132 | 133 | if (this.config.aiProvider === 'llama2') { 134 | this.addEnvVars({ 135 | PLT_AI_MODEL: this.config.aiModel ?? './model.gguf' 136 | }, { overwrite: false }) 137 | } else { 138 | this.addEnvVars({ 139 | PLT_AI_API_KEY: this.config.aiApiKey ?? 'sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' 140 | }, { overwrite: false }) 141 | } 142 | 143 | const packageJson = await this.getStackablePackageJson() 144 | 145 | this.config.dependencies = { 146 | [packageJson.name]: `^${packageJson.version}` 147 | } 148 | 149 | if (this.config.aiProvider === 'llama2') { 150 | this.config.dependencies['node-llama-cpp'] = packageJson.devDependencies['node-llama-cpp'] 151 | } 152 | } 153 | 154 | async _afterPrepare (): Promise { 155 | const packageJson = await this.getStackablePackageJson() 156 | this.addFile({ 157 | path: '', 158 | file: 'global.d.ts', 159 | contents: generateGlobalTypesFile(packageJson.name) 160 | }) 161 | 162 | if (this.config.localSchema as boolean) { 163 | this.addFile({ 164 | path: '', 165 | file: 'stackable.schema.json', 166 | contents: JSON.stringify(schema, null, 2) 167 | }) 168 | } 169 | 170 | if (this.config.plugin !== undefined && this.config.plugin) { 171 | const plugins = generatePlugins(this.config.typescript ?? false) 172 | for (const plugin of plugins) { 173 | this.addFile(plugin) 174 | } 175 | } 176 | } 177 | 178 | async getStackablePackageJson (): Promise { 179 | if (this._packageJson == null) { 180 | const packageJsonPath = this.config.aiWarpPackageJsonPath 181 | const packageJsonFile = await readFile(packageJsonPath, 'utf8') 182 | const packageJson: Partial = JSON.parse(packageJsonFile) 183 | 184 | if (packageJson.name === undefined || packageJson.name === null) { 185 | throw new Error('Missing package name in package.json') 186 | } 187 | 188 | if (packageJson.version === undefined || packageJson.version === null) { 189 | throw new Error('Missing package version in package.json') 190 | } 191 | 192 | this._packageJson = packageJson as PackageJson 193 | return packageJson as PackageJson 194 | } 195 | return this._packageJson 196 | } 197 | 198 | async prepareQuestions (): Promise { 199 | this.questions.push({ 200 | type: 'list', 201 | name: 'aiProvider', 202 | message: 'What AI provider would you like to use?', 203 | default: true, 204 | choices: ['openai', 'mistral', 'azure', 'ollama', 'llama2'] 205 | }) 206 | 207 | this.questions.push({ 208 | type: 'input', 209 | name: 'aiModel', 210 | message: 'What AI model would you like to use?', 211 | default (answers: Record) { 212 | if (answers.aiProvider === 'openai') { 213 | return 'gpt-3.5-turbo' 214 | } else if (answers.aiProvider === 'mistral') { 215 | return 'open-mistral-7b' 216 | } else if (answers.aiProvider === 'azure') { 217 | return 'gpt-35-turbo' 218 | } else if (answers.aiProvider === 'ollama') { 219 | return 'mistral' 220 | } else if (answers.aiProvider === 'llama2') { 221 | return './mymodel.gguf' 222 | } 223 | return 'gpt-3.5-turbo' 224 | } 225 | }) 226 | 227 | this.questions.push({ 228 | type: 'input', 229 | name: 'aiApiKey', 230 | when: (answers: Record) => answers.aiProvider !== 'ollama' && answers.aiProvider !== 'llama2', 231 | message: 'What is your OpenAI/Mistral/Azure API key?', 232 | default: 'sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' 233 | }) 234 | } 235 | } 236 | 237 | export default AiWarpGenerator 238 | export { AiWarpGenerator as Generator } 239 | -------------------------------------------------------------------------------- /lib/schema.ts: -------------------------------------------------------------------------------- 1 | import { readFileSync } from 'node:fs' 2 | import { join, dirname } from 'node:path' 3 | import { schema } from '@platformatic/service' 4 | import { fileURLToPath } from 'node:url' 5 | 6 | let pkgJsonPath: string 7 | if (import.meta.url.endsWith('.js')) { 8 | pkgJsonPath = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'package.json') 9 | } else { 10 | pkgJsonPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json') 11 | } 12 | 13 | const pkgJson: { version: string } = JSON.parse(readFileSync(pkgJsonPath, 'utf8')) 14 | 15 | const aiWarpSchema = { 16 | ...schema.schema, 17 | $id: `https://schemas.platformatic.com/@platformatic/ai-warp/${pkgJson.version}.json`, 18 | title: 'Ai Warp Config', 19 | version: pkgJson.version, 20 | properties: { 21 | ...schema.schema.properties, 22 | module: { type: 'string' }, 23 | showAiWarpHomepage: { 24 | type: 'boolean', 25 | default: true 26 | }, 27 | aiProvider: { 28 | type: 'object', 29 | oneOf: [ 30 | { 31 | properties: { 32 | openai: { 33 | type: 'object', 34 | properties: { 35 | model: { 36 | type: 'string', 37 | enum: [ 38 | 'gpt-4-0125-preview', 39 | 'gpt-4-turbo-preview', 40 | 'gpt-4-1106-preview', 41 | 'gpt-4-vision-preview', 42 | 'gpt-4-1106-vision-preview', 43 | 'gpt-4', 44 | 'gpt-4-0613', 45 | 'gpt-4-32k', 46 | 'gpt-4-32k-0613', 47 | 'gpt-3.5-turbo-0125', 48 | 'gpt-3.5-turbo', 49 | 'gpt-3.5-turbo-1106', 50 | 'gpt-3.5-turbo-instruct', 51 | 'gpt-3.5-turbo-16k', 52 | 'gpt-3.5-turbo-0613', 53 | 'gpt-3.5-turbo-16k-0613' 54 | ] 55 | }, 56 | apiKey: { type: 'string' } 57 | }, 58 | required: ['model', 'apiKey'], 59 | additionalProperties: false 60 | } 61 | }, 62 | required: ['openai'], 63 | additionalProperties: false 64 | }, 65 | { 66 | properties: { 67 | mistral: { 68 | type: 'object', 69 | properties: { 70 | model: { 71 | type: 'string', 72 | enum: [ 73 | 'open-mistral-7b', 74 | 'open-mixtral-8x7b', 75 | 'mistral-small-latest', 76 | 'mistral-medium-latest', 77 | 'mistral-large-latest' 78 | ] 79 | }, 80 | apiKey: { type: 'string' } 81 | }, 82 | required: ['model', 'apiKey'], 83 | additionalProperties: false 84 | } 85 | }, 86 | required: ['mistral'], 87 | additionalProperties: false 88 | }, 89 | { 90 | properties: { 91 | ollama: { 92 | type: 'object', 93 | properties: { 94 | host: { type: 'string' }, 95 | model: { type: 'string' } 96 | }, 97 | required: ['host', 'model'], 98 | additionalProperties: false 99 | } 100 | }, 101 | required: ['ollama'], 102 | additionalProperties: false 103 | }, 104 | { 105 | properties: { 106 | azure: { 107 | type: 'object', 108 | properties: { 109 | endpoint: { type: 'string' }, 110 | apiKey: { type: 'string' }, 111 | deploymentName: { type: 'string' }, 112 | allowInsecureConnections: { 113 | type: 'boolean', 114 | default: false 115 | } 116 | }, 117 | required: ['endpoint', 'apiKey', 'deploymentName'], 118 | additionalProperties: false 119 | } 120 | }, 121 | required: ['azure'], 122 | additionalProperties: false 123 | }, 124 | { 125 | properties: { 126 | llama2: { 127 | type: 'object', 128 | properties: { 129 | modelPath: { type: 'string' } 130 | }, 131 | required: ['modelPath'], 132 | additionalProperties: false 133 | } 134 | }, 135 | required: ['llama2'], 136 | additionalProperties: false 137 | } 138 | ] 139 | }, 140 | promptDecorators: { 141 | type: 'object', 142 | properties: { 143 | prefix: { type: 'string' }, 144 | suffix: { type: 'string' } 145 | }, 146 | additionalProperties: false 147 | }, 148 | auth: { 149 | type: 'object', 150 | properties: { 151 | required: { 152 | type: 'boolean', 153 | description: 'If true, any unauthenticated requests will be blocked', 154 | default: false 155 | }, 156 | // Pulled from https://github.com/platformatic/fastify-user/blob/c7480cef408ea4202087eeb0892730650480c45b/index.d.ts 157 | jwt: { 158 | type: 'object', 159 | properties: { 160 | jwks: { 161 | oneOf: [ 162 | { type: 'boolean' }, 163 | { 164 | type: 'object', 165 | properties: { 166 | max: { type: 'number' }, 167 | ttl: { type: 'number' }, 168 | issuersWhitelist: { 169 | type: 'array', 170 | items: { type: 'string' } 171 | }, 172 | providerDiscovery: { type: 'boolean' }, 173 | jwksPath: { type: 'string' }, 174 | timeout: { type: 'number' } 175 | } 176 | } 177 | ] 178 | }, 179 | // Pulled from https://github.com/fastify/fastify-jwt/blob/77721ccfc9f0ccf1daf477a768eb42827ba48a23/types/jwt.d.ts#L133 180 | secret: { 181 | oneOf: [ 182 | { type: 'string' }, 183 | { 184 | type: 'object', 185 | properties: { 186 | public: { type: 'string' }, 187 | private: { type: 'string' } 188 | }, 189 | required: ['public'] 190 | } 191 | ] 192 | }, 193 | decode: { 194 | type: 'object', 195 | properties: { 196 | complete: { type: 'boolean' }, 197 | // Typo purposeful https://github.com/nearform/fast-jwt/blob/bf0872fb797b60b9e0ffa7f8feb27cdb27a027e6/src/index.d.ts#L116 198 | checkTyp: { type: 'string' } 199 | } 200 | }, 201 | sign: { 202 | type: 'object', 203 | properties: { 204 | expiresIn: { 205 | oneOf: [ 206 | { type: 'number' }, 207 | { type: 'string' } 208 | ] 209 | }, 210 | notBefore: { 211 | oneOf: [ 212 | { type: 'number' }, 213 | { type: 'string' } 214 | ] 215 | }, 216 | key: { type: 'string' } 217 | }, 218 | requires: ['expiresIn', 'notBefore'] 219 | }, 220 | verify: { 221 | type: 'object', 222 | properties: { 223 | maxAge: { 224 | oneOf: [ 225 | { type: 'number' }, 226 | { type: 'string' } 227 | ] 228 | }, 229 | onlyCookie: { type: 'boolean' }, 230 | key: { type: 'string' } 231 | }, 232 | required: ['maxAge', 'onlyCookie'] 233 | }, 234 | cookie: { 235 | type: 'object', 236 | properties: { 237 | cookieName: { type: 'string' }, 238 | signed: { type: 'boolean' } 239 | }, 240 | required: ['cookieName', 'signed'] 241 | }, 242 | messages: { 243 | type: 'object', 244 | properties: { 245 | badRequestErrorMessage: { type: 'string' }, 246 | badCookieRequestErrorMessage: { type: 'string' }, 247 | noAuthorizationInHeaderMessage: { type: 'string' }, 248 | noAuthorizationInCookieMessage: { type: 'string' }, 249 | authorizationTokenExpiredMessage: { type: 'string' }, 250 | authorizationTokenInvalid: { type: 'string' }, 251 | authorizationTokenUntrusted: { type: 'string' }, 252 | authorizationTokenUnsigned: { type: 'string' } 253 | } 254 | }, 255 | jwtDecode: { type: 'string' }, 256 | namespace: { type: 'string' }, 257 | jwtVerify: { type: 'string' }, 258 | jwtSign: { type: 'string' }, 259 | decoratorName: { type: 'string' } 260 | }, 261 | required: ['secret'] 262 | }, 263 | webhook: { 264 | type: 'object', 265 | properties: { 266 | url: { type: 'string' } 267 | }, 268 | required: ['url'] 269 | } 270 | } 271 | }, 272 | rateLimiting: { 273 | type: 'object', 274 | properties: { 275 | // Pulled from https://github.com/fastify/fastify-rate-limit/blob/master/types/index.d.ts#L81 276 | max: { type: 'number' }, 277 | maxByClaims: { 278 | type: 'array', 279 | items: { 280 | type: 'object', 281 | properties: { 282 | claim: { type: 'string' }, 283 | claimValue: { type: 'string' }, 284 | max: { type: 'number' } 285 | }, 286 | additionalProperties: false, 287 | required: ['claim', 'claimValue', 'max'] 288 | } 289 | }, 290 | timeWindow: { 291 | oneOf: [ 292 | { type: 'number' }, 293 | { type: 'string' } 294 | ] 295 | }, 296 | hook: { 297 | type: 'string', 298 | enum: [ 299 | 'onRequest', 300 | 'preParsing', 301 | 'preValidation', 302 | 'preHandler' 303 | ] 304 | }, 305 | cache: { type: 'number' }, 306 | allowList: { 307 | type: 'array', 308 | items: { type: 'string' } 309 | }, 310 | continueExceeding: { type: 'boolean' }, 311 | skipOnError: { type: 'boolean' }, 312 | ban: { type: 'number' }, 313 | enableDraftSpec: { type: 'boolean' } 314 | } 315 | } 316 | }, 317 | required: [ 318 | 'aiProvider' 319 | ] 320 | } 321 | 322 | export { aiWarpSchema as schema } 323 | 324 | if (process.argv.length > 2 && process.argv[2] === '--dump-schema') { 325 | console.log(JSON.stringify(aiWarpSchema, null, 2)) 326 | } 327 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /tests/e2e/api.test.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-floating-promises */ 2 | import { before, after, describe, it } from 'node:test' 3 | import assert from 'node:assert' 4 | import { FastifyInstance } from 'fastify' 5 | import fastifyPlugin from 'fastify-plugin' 6 | import { AiWarpConfig } from '../../config.js' 7 | import { buildAiWarpApp } from '../utils/stackable.js' 8 | import { AZURE_DEPLOYMENT_NAME, AZURE_MOCK_HOST, chatHistoryProvided as azureChatHistoryProvided, resetAzureMock } from '../utils/mocks/azure.js' 9 | import { MOCK_CONTENT_RESPONSE, buildExpectedStreamBodyString } from '../utils/mocks/base.js' 10 | import { OLLAMA_MOCK_HOST, chatHistoryProvided as ollamaChatHistoryProvided, resetOllamaMock } from '../utils/mocks/ollama.js' 11 | import { mockAllProviders } from '../utils/mocks/index.js' 12 | import { chatHistoryProvided as openAiChatHistoryProvided, resetOpenAiMock } from '../utils/mocks/open-ai.js' 13 | import { chatHistoryProvided as mistralChatHistoryProvided, resetMistralMock } from '../utils/mocks/mistral.js' 14 | import stackable, { buildStackable } from '../../index.js' 15 | mockAllProviders() 16 | 17 | const expectedStreamBody = buildExpectedStreamBodyString() 18 | 19 | interface Provider { 20 | name: string 21 | config: AiWarpConfig['aiProvider'] 22 | } 23 | 24 | const providers: Provider[] = [ 25 | { 26 | name: 'OpenAI', 27 | config: { 28 | openai: { 29 | model: 'gpt-3.5-turbo', 30 | apiKey: '' 31 | } 32 | } 33 | }, 34 | { 35 | name: 'Ollama', 36 | config: { 37 | ollama: { 38 | host: OLLAMA_MOCK_HOST, 39 | model: 'some-model' 40 | } 41 | } 42 | }, 43 | { 44 | name: 'Azure', 45 | config: { 46 | azure: { 47 | endpoint: AZURE_MOCK_HOST, 48 | apiKey: 'asd', 49 | deploymentName: AZURE_DEPLOYMENT_NAME, 50 | allowInsecureConnections: true 51 | } 52 | } 53 | }, 54 | { 55 | name: 'Mistral', 56 | config: { 57 | mistral: { 58 | model: 'open-mistral-7b', 59 | apiKey: '' 60 | } 61 | } 62 | } 63 | ] 64 | 65 | // Test the prompt and stream endpoint for each provider 66 | for (const { name, config } of providers) { 67 | describe(name, () => { 68 | let app: FastifyInstance 69 | let port: number 70 | let chunkCallbackCalled = false 71 | before(async () => { 72 | [app, port] = await buildAiWarpApp({ aiProvider: config }) 73 | 74 | await app.register(fastifyPlugin(async () => { 75 | app.ai.preResponseChunkCallback = (_, response) => { 76 | chunkCallbackCalled = true 77 | return response 78 | } 79 | })) 80 | 81 | await app.start() 82 | }) 83 | 84 | after(async () => { 85 | await app.close() 86 | }) 87 | 88 | it('/api/v1/prompt returns expected response', async () => { 89 | const res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 90 | method: 'POST', 91 | headers: { 92 | 'content-type': 'application/json' 93 | }, 94 | body: JSON.stringify({ 95 | prompt: 'asd' 96 | }) 97 | }) 98 | assert.strictEqual(res.headers.get('content-type'), 'application/json; charset=utf-8') 99 | 100 | const body = await res.json() 101 | assert.strictEqual(body.response, MOCK_CONTENT_RESPONSE) 102 | }) 103 | 104 | it('/api/v1/stream returns expected response', async () => { 105 | assert.strictEqual(chunkCallbackCalled, false) 106 | 107 | const res = await fetch(`http://localhost:${port}/api/v1/stream`, { 108 | method: 'POST', 109 | headers: { 110 | 'content-type': 'application/json' 111 | }, 112 | body: JSON.stringify({ 113 | prompt: 'asd' 114 | }) 115 | }) 116 | 117 | assert.strictEqual(res.headers.get('content-type'), 'text/event-stream') 118 | 119 | assert.strictEqual(chunkCallbackCalled, true) 120 | 121 | assert.notStrictEqual(res.body, undefined) 122 | 123 | // eslint-disable-next-line @typescript-eslint/no-non-null-assertion 124 | const reader = res.body!.getReader() 125 | 126 | let body = '' 127 | const decoder = new TextDecoder() 128 | while (true) { 129 | const { value, done } = await reader.read() 130 | if (done !== undefined && done) { 131 | break 132 | } 133 | 134 | body += decoder.decode(value) 135 | } 136 | 137 | assert.strictEqual(body, expectedStreamBody) 138 | }) 139 | }) 140 | } 141 | 142 | it('OpenAI /api/v1/prompt works with chat history', async () => { 143 | resetOpenAiMock() 144 | 145 | const [app, port] = await buildAiWarpApp({ 146 | aiProvider: { 147 | openai: { 148 | model: 'gpt-3.5-turbo', 149 | apiKey: '' 150 | } 151 | } 152 | }) 153 | 154 | await app.start() 155 | try { 156 | const res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 157 | method: 'POST', 158 | headers: { 159 | 'content-type': 'application/json' 160 | }, 161 | body: JSON.stringify({ 162 | prompt: 'asd', 163 | chatHistory: [ 164 | { 165 | prompt: '', 166 | response: '' 167 | } 168 | ] 169 | }) 170 | }) 171 | assert.strictEqual(res.status, 200) 172 | 173 | assert.strictEqual(openAiChatHistoryProvided, true) 174 | } finally { 175 | app.close() 176 | } 177 | }) 178 | 179 | it('Mistral /api/v1/prompt works with chat history', async () => { 180 | resetMistralMock() 181 | 182 | const [app, port] = await buildAiWarpApp({ 183 | aiProvider: { 184 | mistral: { 185 | model: 'mistral-small-latest', 186 | apiKey: '' 187 | } 188 | } 189 | }) 190 | 191 | await app.start() 192 | try { 193 | const res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 194 | method: 'POST', 195 | headers: { 196 | 'content-type': 'application/json' 197 | }, 198 | body: JSON.stringify({ 199 | prompt: 'asd', 200 | chatHistory: [ 201 | { 202 | prompt: '', 203 | response: '' 204 | } 205 | ] 206 | }) 207 | }) 208 | assert.strictEqual(res.status, 200) 209 | 210 | assert.strictEqual(mistralChatHistoryProvided, true) 211 | } finally { 212 | app.close() 213 | } 214 | }) 215 | 216 | it('Ollama /api/v1/prompt works with chat history', async () => { 217 | resetOllamaMock() 218 | 219 | const [app, port] = await buildAiWarpApp({ 220 | aiProvider: { 221 | ollama: { 222 | host: OLLAMA_MOCK_HOST, 223 | model: 'some-model' 224 | } 225 | } 226 | }) 227 | 228 | await app.start() 229 | try { 230 | const res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 231 | method: 'POST', 232 | headers: { 233 | 'content-type': 'application/json' 234 | }, 235 | body: JSON.stringify({ 236 | prompt: 'asd', 237 | chatHistory: [ 238 | { 239 | prompt: '', 240 | response: '' 241 | } 242 | ] 243 | }) 244 | }) 245 | assert.strictEqual(res.status, 200) 246 | 247 | assert.strictEqual(ollamaChatHistoryProvided, true) 248 | } finally { 249 | app.close() 250 | } 251 | }) 252 | 253 | it('Azure /api/v1/prompt works with chat history', async () => { 254 | resetAzureMock() 255 | 256 | const [app, port] = await buildAiWarpApp({ 257 | aiProvider: { 258 | azure: { 259 | endpoint: AZURE_MOCK_HOST, 260 | apiKey: 'asd', 261 | deploymentName: AZURE_DEPLOYMENT_NAME, 262 | allowInsecureConnections: true 263 | } 264 | } 265 | }) 266 | 267 | await app.start() 268 | try { 269 | const res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 270 | method: 'POST', 271 | headers: { 272 | 'content-type': 'application/json' 273 | }, 274 | body: JSON.stringify({ 275 | prompt: 'asd', 276 | chatHistory: [ 277 | { 278 | prompt: '', 279 | response: '' 280 | } 281 | ] 282 | }) 283 | }) 284 | assert.strictEqual(res.status, 200) 285 | 286 | assert.strictEqual(azureChatHistoryProvided, true) 287 | } finally { 288 | app.close() 289 | } 290 | }) 291 | 292 | it('calls the preResponseCallback', async () => { 293 | const [app, port] = await buildAiWarpApp({ 294 | aiProvider: { 295 | openai: { 296 | model: 'gpt-3.5-turbo', 297 | apiKey: '' 298 | } 299 | } 300 | }) 301 | 302 | let callbackCalled = false 303 | await app.register(fastifyPlugin(async () => { 304 | app.ai.preResponseCallback = (_, response) => { 305 | callbackCalled = true 306 | return response + ' modified' 307 | } 308 | })) 309 | 310 | await app.start() 311 | 312 | const res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 313 | method: 'POST', 314 | headers: { 315 | 'content-type': 'application/json' 316 | }, 317 | body: JSON.stringify({ 318 | prompt: 'asd' 319 | }) 320 | }) 321 | 322 | assert.strictEqual(callbackCalled, true) 323 | 324 | const body = await res.json() 325 | assert.strictEqual(body.response, `${MOCK_CONTENT_RESPONSE} modified`) 326 | 327 | await app.close() 328 | }) 329 | 330 | it('provides all paths in OpenAPI', async () => { 331 | const [app, port] = await buildAiWarpApp({ 332 | aiProvider: { 333 | openai: { 334 | model: 'gpt-3.5-turbo', 335 | apiKey: '' 336 | } 337 | } 338 | }) 339 | 340 | await app.start() 341 | 342 | const res = await fetch(`http://localhost:${port}/documentation/json`) 343 | const body = await res.json() 344 | 345 | assert.deepStrictEqual(Object.keys(body.paths), [ 346 | '/api/v1/prompt', 347 | '/api/v1/stream' 348 | ]) 349 | 350 | await app.close() 351 | }) 352 | 353 | it('prompt with wrong JSON', async () => { 354 | const [app, port] = await buildAiWarpApp({ 355 | aiProvider: { 356 | openai: { 357 | model: 'gpt-3.5-turbo', 358 | apiKey: '' 359 | } 360 | } 361 | }) 362 | 363 | await app.start() 364 | 365 | const res = await fetch(`http://localhost:${port}/api/v1/prompt`, { 366 | method: 'POST', 367 | headers: { 368 | 'content-type': 'application/json' 369 | }, 370 | body: JSON.stringify({ 371 | prompt: 'asd' 372 | }).slice(0, 10) 373 | }) 374 | 375 | assert.strictEqual(res.status, 400) 376 | 377 | const body = await res.json() 378 | 379 | assert.deepStrictEqual(body, { 380 | message: 'Unexpected end of JSON input' 381 | }) 382 | 383 | await app.close() 384 | }) 385 | 386 | it('buildStackable', async () => { 387 | const stackable = await buildStackable({ 388 | // @ts-expect-error 389 | config: { 390 | server: { 391 | port: 0, 392 | forceCloseConnections: true, 393 | healthCheck: false, 394 | logger: { 395 | level: 'silent' 396 | } 397 | }, 398 | service: { 399 | openapi: true 400 | }, 401 | aiProvider: { 402 | openai: { 403 | model: 'gpt-3.5-turbo', 404 | apiKey: '' 405 | } 406 | } 407 | } 408 | }) 409 | 410 | await stackable.start({}) 411 | 412 | // @ts-expect-error 413 | const res = await stackable.inject('/documentation/json') 414 | // @ts-expect-error 415 | const body = JSON.parse(res.body) 416 | 417 | assert.deepStrictEqual(Object.keys(body.paths), [ 418 | '/api/v1/prompt', 419 | '/api/v1/stream' 420 | ]) 421 | 422 | await stackable.stop() 423 | }) 424 | 425 | it('stackable.buildStackable', async () => { 426 | const app = await stackable.buildStackable({ 427 | // @ts-expect-error 428 | config: { 429 | server: { 430 | port: 0, 431 | forceCloseConnections: true, 432 | healthCheck: false, 433 | logger: { 434 | level: 'silent' 435 | } 436 | }, 437 | service: { 438 | openapi: true 439 | }, 440 | aiProvider: { 441 | openai: { 442 | model: 'gpt-3.5-turbo', 443 | apiKey: '' 444 | } 445 | } 446 | } 447 | }) 448 | 449 | await app.start({}) 450 | 451 | // @ts-expect-error 452 | const res = await app.inject('/documentation/json') 453 | // @ts-expect-error 454 | const body = JSON.parse(res.body) 455 | 456 | assert.deepStrictEqual(Object.keys(body.paths), [ 457 | '/api/v1/prompt', 458 | '/api/v1/stream' 459 | ]) 460 | 461 | await app.stop() 462 | }) 463 | -------------------------------------------------------------------------------- /config.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable */ 2 | /** 3 | * This file was automatically generated by json-schema-to-typescript. 4 | * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, 5 | * and run json-schema-to-typescript to regenerate this file. 6 | */ 7 | 8 | export interface AiWarpConfig { 9 | server?: { 10 | hostname?: string; 11 | port?: number | string; 12 | pluginTimeout?: number; 13 | healthCheck?: 14 | | boolean 15 | | { 16 | enabled?: boolean; 17 | interval?: number; 18 | [k: string]: unknown; 19 | }; 20 | ignoreTrailingSlash?: boolean; 21 | ignoreDuplicateSlashes?: boolean; 22 | connectionTimeout?: number; 23 | keepAliveTimeout?: number; 24 | maxRequestsPerSocket?: number; 25 | forceCloseConnections?: boolean | string; 26 | requestTimeout?: number; 27 | bodyLimit?: number; 28 | maxParamLength?: number; 29 | disableRequestLogging?: boolean; 30 | exposeHeadRoutes?: boolean; 31 | logger?: 32 | | boolean 33 | | { 34 | level: ( 35 | | ("fatal" | "error" | "warn" | "info" | "debug" | "trace" | "silent") 36 | | { 37 | [k: string]: unknown; 38 | } 39 | ) & 40 | string; 41 | transport?: 42 | | { 43 | target?: string; 44 | options?: { 45 | [k: string]: unknown; 46 | }; 47 | } 48 | | { 49 | targets?: { 50 | target?: string; 51 | options?: { 52 | [k: string]: unknown; 53 | }; 54 | level?: string; 55 | additionalProperties?: never; 56 | [k: string]: unknown; 57 | }[]; 58 | options?: { 59 | [k: string]: unknown; 60 | }; 61 | }; 62 | pipeline?: { 63 | target?: string; 64 | options?: { 65 | [k: string]: unknown; 66 | }; 67 | }; 68 | [k: string]: unknown; 69 | }; 70 | loggerInstance?: { 71 | [k: string]: unknown; 72 | }; 73 | serializerOpts?: { 74 | schema?: { 75 | [k: string]: unknown; 76 | }; 77 | ajv?: { 78 | [k: string]: unknown; 79 | }; 80 | rounding?: "floor" | "ceil" | "round" | "trunc"; 81 | debugMode?: boolean; 82 | mode?: "debug" | "standalone"; 83 | largeArraySize?: number | string; 84 | largeArrayMechanism?: "default" | "json-stringify"; 85 | [k: string]: unknown; 86 | }; 87 | caseSensitive?: boolean; 88 | requestIdHeader?: string | false; 89 | requestIdLogLabel?: string; 90 | jsonShorthand?: boolean; 91 | trustProxy?: boolean | string | string[] | number; 92 | http2?: boolean; 93 | https?: { 94 | allowHTTP1?: boolean; 95 | key: 96 | | string 97 | | { 98 | path?: string; 99 | } 100 | | ( 101 | | string 102 | | { 103 | path?: string; 104 | } 105 | )[]; 106 | cert: 107 | | string 108 | | { 109 | path?: string; 110 | } 111 | | ( 112 | | string 113 | | { 114 | path?: string; 115 | } 116 | )[]; 117 | requestCert?: boolean; 118 | rejectUnauthorized?: boolean; 119 | }; 120 | cors?: { 121 | origin?: 122 | | boolean 123 | | string 124 | | ( 125 | | string 126 | | { 127 | regexp: string; 128 | [k: string]: unknown; 129 | } 130 | )[] 131 | | { 132 | regexp: string; 133 | [k: string]: unknown; 134 | }; 135 | methods?: string[]; 136 | /** 137 | * Comma separated string of allowed headers. 138 | */ 139 | allowedHeaders?: string; 140 | exposedHeaders?: string[] | string; 141 | credentials?: boolean; 142 | maxAge?: number; 143 | preflightContinue?: boolean; 144 | optionsSuccessStatus?: number; 145 | preflight?: boolean; 146 | strictPreflight?: boolean; 147 | hideOptionsRoute?: boolean; 148 | }; 149 | }; 150 | plugins?: { 151 | [k: string]: unknown; 152 | }; 153 | metrics?: 154 | | boolean 155 | | { 156 | port?: number | string; 157 | hostname?: string; 158 | endpoint?: string; 159 | server?: "own" | "parent" | "hide"; 160 | defaultMetrics?: { 161 | enabled: boolean; 162 | }; 163 | auth?: { 164 | username: string; 165 | password: string; 166 | }; 167 | labels?: { 168 | [k: string]: string; 169 | }; 170 | }; 171 | telemetry?: OpenTelemetry; 172 | watch?: 173 | | { 174 | enabled?: boolean | string; 175 | /** 176 | * @minItems 1 177 | */ 178 | allow?: [string, ...string[]]; 179 | ignore?: string[]; 180 | } 181 | | boolean 182 | | string; 183 | $schema?: string; 184 | module?: string; 185 | service?: { 186 | openapi?: 187 | | { 188 | info?: Info; 189 | jsonSchemaDialect?: string; 190 | servers?: Server[]; 191 | paths?: Paths; 192 | webhooks?: { 193 | [k: string]: PathItemOrReference; 194 | }; 195 | components?: Components; 196 | security?: SecurityRequirement[]; 197 | tags?: Tag[]; 198 | externalDocs?: ExternalDocumentation; 199 | /** 200 | * Base URL for the OpenAPI Swagger Documentation 201 | */ 202 | swaggerPrefix?: string; 203 | /** 204 | * Path to an OpenAPI spec file 205 | */ 206 | path?: string; 207 | } 208 | | boolean; 209 | graphql?: 210 | | { 211 | graphiql?: boolean; 212 | } 213 | | boolean; 214 | }; 215 | clients?: { 216 | serviceId?: string; 217 | name?: string; 218 | type?: "openapi" | "graphql"; 219 | path?: string; 220 | schema?: string; 221 | url?: string; 222 | fullResponse?: boolean; 223 | fullRequest?: boolean; 224 | validateResponse?: boolean; 225 | }[]; 226 | showAiWarpHomepage?: boolean; 227 | aiProvider: 228 | | { 229 | openai: { 230 | model: 231 | | "gpt-4-0125-preview" 232 | | "gpt-4-turbo-preview" 233 | | "gpt-4-1106-preview" 234 | | "gpt-4-vision-preview" 235 | | "gpt-4-1106-vision-preview" 236 | | "gpt-4" 237 | | "gpt-4-0613" 238 | | "gpt-4-32k" 239 | | "gpt-4-32k-0613" 240 | | "gpt-3.5-turbo-0125" 241 | | "gpt-3.5-turbo" 242 | | "gpt-3.5-turbo-1106" 243 | | "gpt-3.5-turbo-instruct" 244 | | "gpt-3.5-turbo-16k" 245 | | "gpt-3.5-turbo-0613" 246 | | "gpt-3.5-turbo-16k-0613"; 247 | apiKey: string; 248 | }; 249 | } 250 | | { 251 | mistral: { 252 | model: 253 | | "open-mistral-7b" 254 | | "open-mixtral-8x7b" 255 | | "mistral-small-latest" 256 | | "mistral-medium-latest" 257 | | "mistral-large-latest"; 258 | apiKey: string; 259 | }; 260 | } 261 | | { 262 | ollama: { 263 | host: string; 264 | model: string; 265 | }; 266 | } 267 | | { 268 | azure: { 269 | endpoint: string; 270 | apiKey: string; 271 | deploymentName: string; 272 | allowInsecureConnections?: boolean; 273 | }; 274 | } 275 | | { 276 | llama2: { 277 | modelPath: string; 278 | }; 279 | }; 280 | promptDecorators?: { 281 | prefix?: string; 282 | suffix?: string; 283 | }; 284 | auth?: { 285 | /** 286 | * If true, any unauthenticated requests will be blocked 287 | */ 288 | required?: boolean; 289 | jwt?: { 290 | jwks?: 291 | | boolean 292 | | { 293 | max?: number; 294 | ttl?: number; 295 | issuersWhitelist?: string[]; 296 | providerDiscovery?: boolean; 297 | jwksPath?: string; 298 | timeout?: number; 299 | [k: string]: unknown; 300 | }; 301 | secret: 302 | | string 303 | | { 304 | public: string; 305 | private?: string; 306 | [k: string]: unknown; 307 | }; 308 | decode?: { 309 | complete?: boolean; 310 | checkTyp?: string; 311 | [k: string]: unknown; 312 | }; 313 | sign?: { 314 | expiresIn?: number | string; 315 | notBefore?: number | string; 316 | key?: string; 317 | [k: string]: unknown; 318 | }; 319 | verify?: { 320 | maxAge: number | string; 321 | onlyCookie: boolean; 322 | key?: string; 323 | [k: string]: unknown; 324 | }; 325 | cookie?: { 326 | cookieName: string; 327 | signed: boolean; 328 | [k: string]: unknown; 329 | }; 330 | messages?: { 331 | badRequestErrorMessage?: string; 332 | badCookieRequestErrorMessage?: string; 333 | noAuthorizationInHeaderMessage?: string; 334 | noAuthorizationInCookieMessage?: string; 335 | authorizationTokenExpiredMessage?: string; 336 | authorizationTokenInvalid?: string; 337 | authorizationTokenUntrusted?: string; 338 | authorizationTokenUnsigned?: string; 339 | [k: string]: unknown; 340 | }; 341 | jwtDecode?: string; 342 | namespace?: string; 343 | jwtVerify?: string; 344 | jwtSign?: string; 345 | decoratorName?: string; 346 | [k: string]: unknown; 347 | }; 348 | webhook?: { 349 | url: string; 350 | [k: string]: unknown; 351 | }; 352 | [k: string]: unknown; 353 | }; 354 | rateLimiting?: { 355 | max?: number; 356 | maxByClaims?: { 357 | claim: string; 358 | claimValue: string; 359 | max: number; 360 | }[]; 361 | timeWindow?: number | string; 362 | hook?: "onRequest" | "preParsing" | "preValidation" | "preHandler"; 363 | cache?: number; 364 | allowList?: string[]; 365 | continueExceeding?: boolean; 366 | skipOnError?: boolean; 367 | ban?: number; 368 | enableDraftSpec?: boolean; 369 | [k: string]: unknown; 370 | }; 371 | } 372 | export interface OpenTelemetry { 373 | /** 374 | * The name of the service. Defaults to the folder name if not specified. 375 | */ 376 | serviceName: string; 377 | /** 378 | * The version of the service (optional) 379 | */ 380 | version?: string; 381 | /** 382 | * An array of paths to skip when creating spans. Useful for health checks and other endpoints that do not need to be traced. 383 | */ 384 | skip?: { 385 | /** 386 | * The path to skip. Can be a string or a regex. 387 | */ 388 | path?: string; 389 | /** 390 | * HTTP method to skip 391 | */ 392 | method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH" | "HEAD" | "OPTIONS"; 393 | [k: string]: unknown; 394 | }[]; 395 | exporter?: 396 | | { 397 | type?: "console" | "otlp" | "zipkin" | "memory"; 398 | /** 399 | * Options for the exporter. These are passed directly to the exporter. 400 | */ 401 | options?: { 402 | /** 403 | * The URL to send the traces to. Not used for console or memory exporters. 404 | */ 405 | url?: string; 406 | /** 407 | * Headers to send to the exporter. Not used for console or memory exporters. 408 | */ 409 | headers?: { 410 | [k: string]: unknown; 411 | }; 412 | [k: string]: unknown; 413 | }; 414 | additionalProperties?: never; 415 | [k: string]: unknown; 416 | }[] 417 | | { 418 | type?: "console" | "otlp" | "zipkin" | "memory"; 419 | /** 420 | * Options for the exporter. These are passed directly to the exporter. 421 | */ 422 | options?: { 423 | /** 424 | * The URL to send the traces to. Not used for console or memory exporters. 425 | */ 426 | url?: string; 427 | /** 428 | * Headers to send to the exporter. Not used for console or memory exporters. 429 | */ 430 | headers?: { 431 | [k: string]: unknown; 432 | }; 433 | [k: string]: unknown; 434 | }; 435 | additionalProperties?: never; 436 | [k: string]: unknown; 437 | }; 438 | } 439 | export interface Info { 440 | title: string; 441 | summary?: string; 442 | description?: string; 443 | termsOfService?: string; 444 | contact?: Contact; 445 | license?: License; 446 | version: string; 447 | /** 448 | * This interface was referenced by `Info`'s JSON-Schema definition 449 | * via the `patternProperty` "^x-". 450 | */ 451 | [k: string]: unknown; 452 | } 453 | export interface Contact { 454 | name?: string; 455 | url?: string; 456 | email?: string; 457 | /** 458 | * This interface was referenced by `Contact`'s JSON-Schema definition 459 | * via the `patternProperty` "^x-". 460 | */ 461 | [k: string]: unknown; 462 | } 463 | export interface License { 464 | name: string; 465 | identifier?: string; 466 | url?: string; 467 | /** 468 | * This interface was referenced by `License`'s JSON-Schema definition 469 | * via the `patternProperty` "^x-". 470 | */ 471 | [k: string]: unknown; 472 | } 473 | export interface Server { 474 | url: string; 475 | description?: string; 476 | variables?: { 477 | [k: string]: ServerVariable; 478 | }; 479 | /** 480 | * This interface was referenced by `Server`'s JSON-Schema definition 481 | * via the `patternProperty` "^x-". 482 | */ 483 | [k: string]: unknown; 484 | } 485 | export interface ServerVariable { 486 | /** 487 | * @minItems 1 488 | */ 489 | enum?: [string, ...string[]]; 490 | default: string; 491 | description?: string; 492 | /** 493 | * This interface was referenced by `ServerVariable`'s JSON-Schema definition 494 | * via the `patternProperty` "^x-". 495 | */ 496 | [k: string]: unknown; 497 | } 498 | export interface Paths { 499 | [k: string]: PathItem; 500 | } 501 | /** 502 | * This interface was referenced by `Paths`'s JSON-Schema definition 503 | * via the `patternProperty` "^/". 504 | */ 505 | export interface PathItem { 506 | summary?: string; 507 | description?: string; 508 | servers?: Server[]; 509 | parameters?: ParameterOrReference[]; 510 | get?: Operation; 511 | put?: Operation; 512 | post?: Operation; 513 | delete?: Operation; 514 | options?: Operation; 515 | head?: Operation; 516 | patch?: Operation; 517 | trace?: Operation; 518 | /** 519 | * This interface was referenced by `PathItem`'s JSON-Schema definition 520 | * via the `patternProperty` "^x-". 521 | */ 522 | [k: string]: unknown; 523 | } 524 | export interface ParameterOrReference { 525 | [k: string]: unknown; 526 | } 527 | export interface Operation { 528 | tags?: string[]; 529 | summary?: string; 530 | description?: string; 531 | externalDocs?: ExternalDocumentation; 532 | operationId?: string; 533 | parameters?: ParameterOrReference[]; 534 | requestBody?: RequestBodyOrReference; 535 | responses?: Responses; 536 | callbacks?: { 537 | [k: string]: CallbacksOrReference; 538 | }; 539 | security?: SecurityRequirement[]; 540 | servers?: Server[]; 541 | /** 542 | * This interface was referenced by `Operation`'s JSON-Schema definition 543 | * via the `patternProperty` "^x-". 544 | */ 545 | [k: string]: unknown; 546 | } 547 | export interface ExternalDocumentation { 548 | description?: string; 549 | url: string; 550 | /** 551 | * This interface was referenced by `ExternalDocumentation`'s JSON-Schema definition 552 | * via the `patternProperty` "^x-". 553 | */ 554 | [k: string]: unknown; 555 | } 556 | export interface RequestBodyOrReference { 557 | [k: string]: unknown; 558 | } 559 | export interface Responses { 560 | [k: string]: ResponseOrReference; 561 | } 562 | export interface ResponseOrReference { 563 | [k: string]: unknown; 564 | } 565 | export interface CallbacksOrReference { 566 | [k: string]: unknown; 567 | } 568 | export interface SecurityRequirement { 569 | [k: string]: string[]; 570 | } 571 | export interface PathItemOrReference { 572 | [k: string]: unknown; 573 | } 574 | export interface Components { 575 | schemas?: { 576 | [k: string]: unknown; 577 | }; 578 | responses?: { 579 | [k: string]: ResponseOrReference; 580 | }; 581 | parameters?: { 582 | [k: string]: ParameterOrReference; 583 | }; 584 | examples?: { 585 | [k: string]: ExampleOrReference; 586 | }; 587 | requestBodies?: { 588 | [k: string]: RequestBodyOrReference; 589 | }; 590 | headers?: { 591 | [k: string]: HeaderOrReference; 592 | }; 593 | securitySchemes?: { 594 | [k: string]: SecuritySchemeOrReference; 595 | }; 596 | links?: { 597 | [k: string]: LinkOrReference; 598 | }; 599 | callbacks?: { 600 | [k: string]: CallbacksOrReference; 601 | }; 602 | pathItems?: { 603 | [k: string]: PathItemOrReference; 604 | }; 605 | /** 606 | * This interface was referenced by `Components`'s JSON-Schema definition 607 | * via the `patternProperty` "^x-". 608 | */ 609 | [k: string]: unknown; 610 | } 611 | export interface ExampleOrReference { 612 | [k: string]: unknown; 613 | } 614 | export interface HeaderOrReference { 615 | [k: string]: unknown; 616 | } 617 | export interface SecuritySchemeOrReference { 618 | [k: string]: unknown; 619 | } 620 | export interface LinkOrReference { 621 | [k: string]: unknown; 622 | } 623 | export interface Tag { 624 | name: string; 625 | description?: string; 626 | externalDocs?: ExternalDocumentation; 627 | /** 628 | * This interface was referenced by `Tag`'s JSON-Schema definition 629 | * via the `patternProperty` "^x-". 630 | */ 631 | [k: string]: unknown; 632 | } 633 | -------------------------------------------------------------------------------- /static/scripts/chat.js: -------------------------------------------------------------------------------- 1 | let promptLock = false 2 | const messages = [] 3 | const messagesContainer = document.getElementById('messages') 4 | const promptInput = document.getElementById('prompt-input') 5 | const promptButton = document.getElementById('prompt-button') 6 | 7 | promptButton.onclick = () => { 8 | handlePrompt(promptInput.value).catch(err => { 9 | throw err 10 | }) 11 | } 12 | 13 | /** 14 | * @param {KeyboardEvent} event 15 | */ 16 | promptInput.onkeydown = (event) => { 17 | if (event.key === 'Enter' && !event.shiftKey) { 18 | handlePrompt(promptInput.value).catch(err => { 19 | throw err 20 | }) 21 | } 22 | } 23 | 24 | const searchParams = new URL(document.location.toString()).searchParams 25 | if (searchParams.has('prompt')) { 26 | handlePrompt(searchParams.get('prompt')).catch(err => { 27 | throw err 28 | }) 29 | } 30 | 31 | /** 32 | * @param {string} prompt 33 | */ 34 | async function handlePrompt (prompt) { 35 | if (prompt === '' || promptLock) { 36 | return 37 | } 38 | promptInput.value = '' 39 | 40 | const message = { 41 | prompt, 42 | response: [], 43 | responseIndex: 0 44 | } 45 | messages.push(message) 46 | drawMessage(message, true, false) 47 | 48 | await promptAiWarp(message) 49 | drawMessage(message, false, true) 50 | } 51 | 52 | /** 53 | * @param {{ 54 | * prompt: string, 55 | * response: ReadableStream[], 56 | * responseIndex: number, 57 | * errored: boolean 58 | * }} message 59 | */ 60 | async function promptAiWarp (message) { 61 | promptLock = true 62 | promptButton.setAttribute('disabled', '') 63 | 64 | let chatHistoryStartIndex 65 | if (messages.length >= 11) { 66 | chatHistoryStartIndex = messages.length - 12 67 | } else { 68 | chatHistoryStartIndex = 0 69 | } 70 | 71 | // Only send the previous 10 messages to abide by token limits. We also 72 | // don't want to sent the latest message, since that's the one we're getting 73 | // the response to 74 | const chatHistory = [] 75 | for (let i = chatHistoryStartIndex; i < messages.length - 1; i++) { 76 | const previousMessage = messages[i] 77 | chatHistory.push({ 78 | prompt: previousMessage.prompt, 79 | response: previousMessage.response[previousMessage.responseIndex] 80 | }) 81 | } 82 | 83 | try { 84 | const res = await fetch('/api/v1/stream', { 85 | method: 'POST', 86 | headers: { 87 | 'content-type': 'application/json' 88 | }, 89 | body: JSON.stringify({ 90 | prompt: message.prompt, 91 | chatHistory 92 | }) 93 | }) 94 | if (res.status !== 200) { 95 | const { message, code } = await res.json() 96 | throw new Error(`AI Warp error: ${message} (${code})`) 97 | } 98 | 99 | message.response[message.responseIndex] = res.body 100 | } catch (err) { 101 | promptLock = false 102 | promptButton.removeAttribute('disabled') 103 | message.errored = true 104 | console.error(err) 105 | } 106 | } 107 | 108 | /** 109 | * @param {{ 110 | * prompt: string, 111 | * response: (string[] | ReadableStream)[], 112 | * responseIndex: number, 113 | * errored: boolean | undefined 114 | * }} message 115 | * @param {boolean} drawPrompt 116 | * @param {boolean} drawResponse 117 | */ 118 | function drawMessage (message, drawPrompt, drawResponse) { 119 | if (drawPrompt) { 120 | drawPromptMessage(message) 121 | } 122 | if (drawResponse) { 123 | drawResponseMessage(message) 124 | } 125 | } 126 | 127 | /** 128 | * @param {{ prompt: string }} message 129 | */ 130 | function drawPromptMessage (message) { 131 | const element = document.createElement('div') 132 | element.classList.add('message') 133 | element.appendChild(drawMessageAvatar('prompt')) 134 | element.appendChild(drawMessageContents('prompt', message)) 135 | 136 | messagesContainer.appendChild(element) 137 | } 138 | 139 | /** 140 | * @param {{ 141 | * prompt: string, 142 | * response: (string[] | ReadableStream)[], 143 | * responseIndex: number, 144 | * errored: boolean | undefined 145 | * }} message 146 | */ 147 | function drawResponseMessage (message) { 148 | const element = document.createElement('div') 149 | element.classList.add('message') 150 | element.appendChild(drawMessageAvatar('response')) 151 | element.appendChild(drawMessageContents('response', message)) 152 | 153 | messagesContainer.appendChild(element) 154 | } 155 | 156 | /** 157 | * @param {'prompt' | 'response'} type 158 | * @returns {HTMLDivElement} 159 | */ 160 | function drawMessageAvatar (type) { 161 | const element = document.createElement('div') 162 | element.classList.add('message-avatar') 163 | 164 | const img = document.createElement('img') 165 | if (type === 'prompt') { 166 | img.setAttribute('src', '/images/avatars/you.svg') 167 | img.setAttribute('alt', 'You') 168 | } else { 169 | img.setAttribute('src', '/images/avatars/platformatic.svg') 170 | img.setAttribute('alt', 'Platformatic Ai-Warp') 171 | } 172 | element.appendChild(img) 173 | 174 | return element 175 | } 176 | 177 | /** 178 | * @param {'prompt' | 'response'} type 179 | * @param {{ 180 | * prompt: string, 181 | * response: (string[] | ReadableStream)[], 182 | * responseIndex: number, 183 | * errored: boolean | undefined 184 | * }} message 185 | * @returns {HTMLDivElement} 186 | */ 187 | function drawMessageContents (type, message) { 188 | const element = document.createElement('div') 189 | element.classList.add('message-contents') 190 | 191 | const author = document.createElement('p') 192 | author.classList.add('message-author') 193 | author.innerHTML = type === 'prompt' ? 'You' : 'Platformatic Ai-Warp' 194 | element.appendChild(author) 195 | 196 | if (message.errored) { 197 | element.appendChild(drawErrorMessageContents(message)) 198 | } else if (type === 'prompt') { 199 | element.appendChild(drawPromptMessageContents(type, message)) 200 | element.appendChild(drawPromptMessageOptions(message)) 201 | } else { 202 | // ReadableStream doesn't have a length property 203 | if (message.response[message.responseIndex].length !== undefined) { 204 | drawCompletedResponseMessageContents(element, message) 205 | element.appendChild(drawResponseMessageOptions(message)) 206 | } else { 207 | drawStreamedMessageContents(element, message) 208 | .then(() => element.appendChild(drawResponseMessageOptions(message))) 209 | .catch(err => { 210 | throw err 211 | }) 212 | } 213 | } 214 | 215 | return element 216 | } 217 | 218 | /** 219 | * @returns {HTMLParagraphElement} 220 | */ 221 | function drawErrorMessageContents () { 222 | const element = document.createElement('p') 223 | element.classList.add('message-error') 224 | element.innerHTML = 'Error Something went wrong. If this issue persists please contact us at support@platformatic.dev' 225 | 226 | return element 227 | } 228 | 229 | /** 230 | * @param {'prompt' | 'response'} type 231 | * @param {{ prompt: string }} message 232 | * @returns {HTMLParagraphElement} 233 | */ 234 | function drawPromptMessageContents (type, message) { 235 | const element = document.createElement('p') 236 | element.appendChild(document.createTextNode(message.prompt)) 237 | 238 | return element 239 | } 240 | 241 | /** 242 | * @param {HTMLDivElement} parent 243 | * @param {{ prompt: string, response: string[][], responseIndex: number }} message 244 | */ 245 | function drawCompletedResponseMessageContents (parent, message) { 246 | let i = 0 247 | do { 248 | const element = document.createElement('p') 249 | element.appendChild(document.createTextNode(message.response[message.responseIndex][i])) 250 | parent.appendChild(element) 251 | i++ 252 | } while (i < message.response[message.responseIndex].length) 253 | } 254 | 255 | /** 256 | * @param {HTMLDivElement} parent 257 | * @param {{ prompt: string, response: ReadableStream, responseIndex: number }} message 258 | */ 259 | async function drawStreamedMessageContents (parent, message) { 260 | let fullResponse = '' 261 | let current = document.createElement('p') 262 | let newLine = true 263 | parent.appendChild(current) 264 | 265 | const parser = new SSEParser(message.response[message.responseIndex]) 266 | while (true) { 267 | const tokens = await parser.pull() 268 | if (tokens === undefined) { 269 | break 270 | } 271 | 272 | const tokenString = escapeHtml(tokens.join('')) 273 | fullResponse += tokenString 274 | 275 | const lines = tokenString.split('\n') 276 | 277 | if (newLine) { 278 | lines[0] = addNonBreakingSpaces(lines[0]) 279 | newLine = false 280 | } 281 | 282 | // If there are is only one line, we can just append it to the current paragraph, 283 | // otherwise we need to create a new paragraph for each line 284 | current.innerHTML += lines[0] 285 | current.scrollIntoView(false) 286 | 287 | for (let i = 1; i < lines.length; i++) { 288 | current = document.createElement('p') 289 | parent.appendChild(current) 290 | current.scrollIntoView(false) 291 | lines[i] = addNonBreakingSpaces(lines[i]) 292 | current.innerHTML += lines[i] 293 | newLine = true 294 | } 295 | } 296 | 297 | message.response[message.responseIndex] = [fullResponse] 298 | 299 | promptLock = false 300 | promptButton.removeAttribute('disabled') 301 | } 302 | 303 | /** 304 | * @param {string} message 305 | * @returns {string} 306 | */ 307 | function addNonBreakingSpaces (str) { 308 | return str.replace(/^ +/g, (spaces) => { 309 | return spaces.split('').map(() => ' ').join('') 310 | }) 311 | } 312 | 313 | /** 314 | * @param {{ prompt: string, response: string, errored: boolean | undefined }} message 315 | * @returns {HTMLParagraphElement} 316 | */ 317 | function drawPromptMessageOptions (message) { 318 | const element = document.createElement('div') 319 | element.classList.add('message-options') 320 | 321 | const rightAlignedElements = document.createElement('div') 322 | rightAlignedElements.classList.add('message-options-right') 323 | rightAlignedElements.appendChild(drawEditPromptButton(element, message)) 324 | element.appendChild(rightAlignedElements) 325 | 326 | return element 327 | } 328 | 329 | /** 330 | * @param {HTMLDivElement} parent 331 | * @param {{ prompt: string }} message 332 | * @returns {HTMLButtonElement} 333 | */ 334 | function drawEditPromptButton (parent, message) { 335 | const element = document.createElement('button') 336 | element.onclick = () => { 337 | // Set the prompt text to be editable 338 | parent.parentNode.children.item(1).setAttribute('contenteditable', 'true') 339 | 340 | parent.innerHTML = '' 341 | parent.appendChild(drawCancelPromptEditButton(parent, message)) 342 | parent.appendChild(drawSubmitPromptEditButton(parent, message)) 343 | } 344 | 345 | const icon = document.createElement('img') 346 | icon.setAttribute('src', '/images/icons/edit.svg') 347 | icon.setAttribute('alt', 'Edit') 348 | element.appendChild(icon) 349 | 350 | return element 351 | } 352 | 353 | /** 354 | * @param {HTMLDivElement} parent 355 | * @param {{ prompt: string }} message 356 | * @returns {HTMLButtonElement} 357 | */ 358 | function drawCancelPromptEditButton (parent, message) { 359 | const element = document.createElement('button') 360 | element.classList.add('cancel-prompt-edit-button') 361 | element.innerHTML = 'Cancel' 362 | 363 | element.onclick = () => { 364 | const promptParagraph = parent.parentNode.children.item(1) 365 | promptParagraph.setAttribute('contenteditable', false) 366 | promptParagraph.innerHTML = message.prompt 367 | 368 | parent.parentNode.appendChild(drawPromptMessageOptions(message)) 369 | parent.remove() 370 | } 371 | 372 | return element 373 | } 374 | 375 | /** 376 | * @param {HTMLDivElement} parent 377 | * @param {{ prompt: string }} message 378 | * @returns {HTMLButtonElement} 379 | */ 380 | function drawSubmitPromptEditButton (parent, message) { 381 | const element = document.createElement('button') 382 | element.classList.add('submit-prompt-edit-button') 383 | element.innerHTML = 'Save and submit' 384 | 385 | element.onclick = () => { 386 | if (promptLock) { 387 | return 388 | } 389 | 390 | const promptParagraph = parent.parentNode.children.item(1) 391 | const newPrompt = promptParagraph.innerHTML 392 | if (newPrompt === message.prompt) { 393 | // No change 394 | return 395 | } 396 | 397 | message.prompt = newPrompt 398 | 399 | promptAiWarp(message) 400 | .catch(err => { 401 | throw err 402 | }) 403 | .finally(() => { 404 | redrawMessages() 405 | }) 406 | } 407 | 408 | return element 409 | } 410 | 411 | /** 412 | * @param {{ prompt: string, response: (string[] | ReadableStream)[], responseIndex: number }} message 413 | * @returns {HTMLDivElement} 414 | */ 415 | function drawResponseMessageOptions (message) { 416 | const element = document.createElement('div') 417 | element.classList.add('message-options') 418 | 419 | if (message.response.length > 1) { 420 | element.appendChild(drawResponseIndexSelector(message)) 421 | } 422 | 423 | const rightAlignedElements = document.createElement('div') 424 | rightAlignedElements.classList.add('message-options-right') 425 | rightAlignedElements.appendChild(drawRegenerateResponseButton(message)) 426 | rightAlignedElements.appendChild(drawCopyResponseButton(message)) 427 | element.appendChild(rightAlignedElements) 428 | 429 | return element 430 | } 431 | 432 | /** 433 | * @param {{ response: (string[] | ReadableStream)[], responseIndex: number }} message 434 | * @returns {HTMLDivElement} 435 | */ 436 | function drawResponseIndexSelector (message) { 437 | const element = document.createElement('div') 438 | element.classList.add('response-index-selector') 439 | 440 | const leftArrow = document.createElement('button') 441 | const leftArrowIcon = document.createElement('img') 442 | leftArrowIcon.setAttribute('src', '/images/icons/arrow-left.svg') 443 | leftArrowIcon.setAttribute('alt', 'Previous') 444 | leftArrow.appendChild(leftArrowIcon) 445 | element.appendChild(leftArrow) 446 | 447 | leftArrow.onclick = () => { 448 | if (message.responseIndex === 0) { 449 | return 450 | } 451 | 452 | message.responseIndex-- 453 | redrawMessages() 454 | } 455 | 456 | const positionText = document.createElement('p') 457 | positionText.innerHTML = `${message.responseIndex + 1}/${message.response.length}` 458 | element.appendChild(positionText) 459 | 460 | const rightArrow = document.createElement('button') 461 | const rightArrowIcon = document.createElement('img') 462 | rightArrowIcon.setAttribute('src', '/images/icons/arrow-right.svg') 463 | rightArrowIcon.setAttribute('alt', 'Next') 464 | rightArrow.appendChild(rightArrowIcon) 465 | element.appendChild(rightArrow) 466 | 467 | rightArrow.onclick = () => { 468 | if (message.responseIndex + 1 >= message.response.length) { 469 | return 470 | } 471 | 472 | message.responseIndex++ 473 | redrawMessages() 474 | } 475 | 476 | return element 477 | } 478 | 479 | /** 480 | * @param {{ prompt: string, responseIndex: number }} message 481 | * @returns {HTMLButtonElement} 482 | */ 483 | function drawRegenerateResponseButton (message) { 484 | const element = document.createElement('button') 485 | element.onclick = () => { 486 | message.responseIndex++ 487 | promptAiWarp(message) 488 | .catch(err => { 489 | throw err 490 | }) 491 | .finally(() => { 492 | redrawMessages() 493 | }) 494 | } 495 | 496 | const icon = document.createElement('img') 497 | icon.setAttribute('src', '/images/icons/regenerate.svg') 498 | icon.setAttribute('alt', 'Regenerate') 499 | element.appendChild(icon) 500 | 501 | return element 502 | } 503 | 504 | /** 505 | * @param {{ response: string[], responseIndex: number }} message 506 | * @returns {HTMLButtonElement} 507 | */ 508 | function drawCopyResponseButton (message) { 509 | const element = document.createElement('button') 510 | 511 | const icon = document.createElement('img') 512 | icon.setAttribute('src', '/images/icons/copy.svg') 513 | icon.setAttribute('alt', 'Copy') 514 | element.appendChild(icon) 515 | 516 | element.onclick = () => { 517 | navigator.clipboard.writeText(message.response[message.responseIndex]) 518 | 519 | icon.setAttribute('src', '/images/icons/checkmark.svg') 520 | setTimeout(() => { 521 | icon.setAttribute('src', '/images/icons/copy.svg') 522 | }, 2000) 523 | } 524 | 525 | return element 526 | } 527 | 528 | function redrawMessages () { 529 | messagesContainer.innerHTML = '' 530 | for (const message of messages) { 531 | drawMessage(message, true, true) 532 | } 533 | } 534 | 535 | /** 536 | * Parser for server sent events returned by the streaming endpoint 537 | */ 538 | class SSEParser { 539 | /** 540 | * @param {ReadableStream} stream 541 | */ 542 | constructor (stream) { 543 | this.reader = stream.getReader() 544 | this.decoder = new TextDecoder() 545 | } 546 | 547 | /** 548 | * @returns {string[] | undefined} Undefined at the end of the stream 549 | */ 550 | async pull () { 551 | const { done, value } = await this.reader.read() 552 | if (done) { 553 | return undefined 554 | } 555 | 556 | const decodedValue = this.decoder.decode(value) 557 | const lines = decodedValue.split('\n') 558 | 559 | const tokens = [] 560 | let i = 0 561 | while (i < lines.length) { 562 | const line = lines[i] 563 | if (line.length === 0) { 564 | i++ 565 | continue 566 | } 567 | 568 | if (!line.startsWith('event: ')) { 569 | throw new Error(`Unexpected event type line: ${line}`) 570 | } 571 | 572 | const dataLine = lines[i + 1] 573 | if (!dataLine.startsWith('data: ')) { 574 | throw new Error(`Unexpected data line: ${dataLine}`) 575 | } 576 | 577 | const eventType = line.substring('event: '.length) 578 | const data = dataLine.substring('data: '.length) 579 | const json = JSON.parse(data) 580 | if (eventType === 'content') { 581 | const { response } = json 582 | tokens.push(response) 583 | } else if (eventType === 'error') { 584 | const { message, code } = data 585 | throw new Error(`AI Warp Error: ${message} (${code})`) 586 | } 587 | 588 | i += 2 589 | } 590 | 591 | return tokens 592 | } 593 | } 594 | 595 | function escapeHtml (str) { 596 | return str.replace( 597 | /[&<>'"]/g, 598 | tag => 599 | ({ 600 | '&': '&', 601 | '<': '<', 602 | '>': '>', 603 | "'": ''', 604 | '"': '"' 605 | }[tag] || tag) 606 | ) 607 | } 608 | --------------------------------------------------------------------------------