├── .eslintignore ├── .prettierignore ├── src ├── types │ └── index.ts ├── utils │ ├── context.ts │ ├── openapi │ │ ├── content-type.ts │ │ ├── index.ts │ │ ├── required-schema.ts │ │ ├── tokenizer.ts │ │ ├── merge-schema.ts │ │ ├── dereference-path.ts │ │ ├── deepen-schema.ts │ │ ├── chunk-schema.ts │ │ ├── operation-tokenizer.ts │ │ └── operation.ts │ ├── template.ts │ ├── logger.ts │ ├── benchmark.ts │ └── alphabet.ts ├── index.ts ├── llm │ └── index.ts ├── embeddings │ └── index.ts └── agent │ ├── code-gen.ts │ ├── context-processor.ts │ ├── external-resource-evaluator.ts │ ├── index.ts │ └── external-resource-directory.ts ├── .vscode ├── settings.json └── launch.json ├── demo └── context.json ├── bun.lockb ├── public └── images │ └── logo.jpg ├── tsconfig.json ├── tests ├── tsconfig.json ├── openapi │ └── operation-tokenizer.test.ts └── __specs__ │ └── test_v3.json ├── tsconfig.module.json ├── .github └── workflows │ └── test.yml ├── .eslintrc.cjs ├── .prettierrc.cjs ├── LICENSE ├── package.json ├── tsconfig.base.json ├── .gitignore └── README.md /.eslintignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/types/index.ts: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | } -------------------------------------------------------------------------------- /demo/context.json: -------------------------------------------------------------------------------- 1 | { 2 | "username": { "type": "string" } 3 | } 4 | -------------------------------------------------------------------------------- /bun.lockb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intervenehq/parser/HEAD/bun.lockb -------------------------------------------------------------------------------- /public/images/logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intervenehq/parser/HEAD/public/images/logo.jpg -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.base.json", 3 | "compilerOptions": { 4 | "rootDir": "src", 5 | "module": "commonjs", 6 | "target": "es2017", 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tests/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.base.json", 3 | "compilerOptions": { 4 | "noEmit": true, 5 | "module": "ESNext" 6 | }, 7 | "include": ["**/*.ts", "../src/**/*.ts"] 8 | } 9 | -------------------------------------------------------------------------------- /tsconfig.module.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig", 3 | "compilerOptions": { 4 | "target": "esnext", 5 | "outDir": "lib/module", 6 | "module": "esnext" 7 | }, 8 | "exclude": ["node_modules/**"] 9 | } 10 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test-workflow 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | test-job: 8 | name: test-job 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | - uses: oven-sh/setup-bun@v1 13 | 14 | - run: bun install 15 | - run: bun test 16 | -------------------------------------------------------------------------------- /src/utils/context.ts: -------------------------------------------------------------------------------- 1 | import { JsonObject } from 'type-fest'; 2 | 3 | function stringifyContext(context: JsonObject | null) { 4 | if (!context) return ''; 5 | 6 | return Object.entries(context) 7 | .map(([key, value]) => { 8 | return '"' + key + '": ' + JSON.stringify(value); 9 | }) 10 | .join('\n'); 11 | } 12 | 13 | export { stringifyContext }; 14 | -------------------------------------------------------------------------------- /src/utils/openapi/content-type.ts: -------------------------------------------------------------------------------- 1 | export function getDefaultContentType(types: string[]) { 2 | if (types.includes('application/json')) return 'application/json'; 3 | if (types.includes('application/x-www-form-urlencoded')) 4 | return 'application/x-www-form-urlencoded'; 5 | if (types.includes('multipart/form-data')) return 'multipart/form-data'; 6 | 7 | return types[0]; 8 | } 9 | -------------------------------------------------------------------------------- /src/utils/template.ts: -------------------------------------------------------------------------------- 1 | import Handlebars from 'handlebars'; 2 | 3 | export const t = ( 4 | template: string[], 5 | data?: T, 6 | options?: { delim?: string } & CompileOptions, 7 | ): string => { 8 | const delim = options?.delim ?? '\n'; 9 | 10 | const compiled = Handlebars.compile(template.join(delim), { 11 | noEscape: true, 12 | ...options, 13 | }); 14 | 15 | return compiled(data); 16 | }; 17 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "command": "npm start", 9 | "name": "Run npm start", 10 | "request": "launch", 11 | "type": "node-terminal" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /src/utils/logger.ts: -------------------------------------------------------------------------------- 1 | import type { Options as OraOptions } from 'ora'; 2 | 3 | export default interface Logger { 4 | log: (...args: any[]) => Promise | void; 5 | info: (...args: any[]) => Promise | void; 6 | warn: (...args: any[]) => Promise | void; 7 | error: (...args: any[]) => Promise | void; 8 | showLoader?: (options: OraOptions) => Promise | void; 9 | hideLoader?: () => Promise | void; 10 | } 11 | -------------------------------------------------------------------------------- /src/utils/benchmark.ts: -------------------------------------------------------------------------------- 1 | import Logger from './logger'; 2 | 3 | export async function benchmark( 4 | name: string, 5 | fn: () => Promise, 6 | logger: Logger, 7 | ) { 8 | const start = Date.now(); 9 | await logger.info(`Starting '${name}'`); 10 | 11 | const result = await fn(); 12 | 13 | const end = Date.now(); 14 | const duration = end - start; 15 | await logger.info(`Finished '${name}' in ${duration}ms`); 16 | 17 | return result; 18 | } 19 | -------------------------------------------------------------------------------- /src/utils/alphabet.ts: -------------------------------------------------------------------------------- 1 | import Handlebars from 'handlebars'; 2 | 3 | export const ALPHABET = [ 4 | 'a', 5 | 'b', 6 | 'c', 7 | 'd', 8 | 'e', 9 | 'f', 10 | 'g', 11 | 'h', 12 | 'i', 13 | 'j', 14 | 'k', 15 | 'l', 16 | 'm', 17 | 'n', 18 | 'o', 19 | 'p', 20 | 'q', 21 | 'r', 22 | 's', 23 | 't', 24 | 'u', 25 | 'v', 26 | 'w', 27 | 'x', 28 | 'y', 29 | 'z', 30 | ]; 31 | 32 | Handlebars.registerHelper('getAlphabet', function (index) { 33 | return ALPHABET[index]; 34 | }); 35 | -------------------------------------------------------------------------------- /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | /* eslint-env node */ 2 | /** @type {import("eslint").Linter.Config} */ 3 | module.exports = { 4 | extends: [ 5 | 'eslint:recommended', 6 | 'plugin:@typescript-eslint/recommended-type-checked', 7 | 'plugin:prettier/recommended', 8 | ], 9 | parser: '@typescript-eslint/parser', 10 | plugins: ['@typescript-eslint'], 11 | ignorePatterns: ['.eslintrc.cjs'], 12 | root: true, 13 | parserOptions: { 14 | project: true, 15 | tsconfigRootDir: __dirname, 16 | }, 17 | rules: { 18 | '@typescript-eslint/no-explicit-any': 'off', 19 | }, 20 | }; 21 | -------------------------------------------------------------------------------- /.prettierrc.cjs: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | 3 | /** @type {import("@ianvs/prettier-plugin-sort-imports").PrettierConfig} */ 4 | module.exports = { 5 | // Standard prettier options 6 | singleQuote: true, 7 | semi: true, 8 | // Since prettier 3.0, manually specifying plugins is required 9 | plugins: ['@ianvs/prettier-plugin-sort-imports'], 10 | // This plugin's options 11 | importOrder: [ 12 | '', // Node.js built-in modules 13 | '', 14 | '', 15 | '^src/(.*)$', 16 | '^../(.*)$', 17 | '', 18 | '^../utils/(.*)$', 19 | '', 20 | '^[./]', 21 | ], 22 | importOrderParserPlugins: ['typescript', 'jsx', 'decorators-legacy'], 23 | importOrderTypeScriptVersion: '5.0.0', 24 | }; 25 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import Parser from './agent'; 2 | import CodeGenerator, { CodeGenLanguage } from './agent/code-gen'; 3 | import ExternalResourceDirectory, { 4 | APIMatch, 5 | } from './agent/external-resource-directory'; 6 | import ExternalResourceEvaluator from './agent/external-resource-evaluator'; 7 | import { benchmark } from './utils/benchmark'; 8 | import Logger from './utils/logger'; 9 | import { tokenizedLength } from './utils/openapi'; 10 | 11 | export * from './llm'; 12 | 13 | export * from './embeddings/index'; 14 | 15 | export { 16 | Parser, 17 | ExternalResourceDirectory, 18 | ExternalResourceEvaluator, 19 | CodeGenLanguage, 20 | CodeGenerator, 21 | Logger, 22 | APIMatch, 23 | benchmark, 24 | tokenizedLength, 25 | }; 26 | -------------------------------------------------------------------------------- /src/utils/openapi/index.ts: -------------------------------------------------------------------------------- 1 | import { encode } from 'gpt-tokenizer'; 2 | import { OpenAPIV2, OpenAPIV3, OpenAPIV3_1 } from 'openapi-types'; 3 | import { JsonValue } from 'type-fest'; 4 | 5 | export type ExcludeReference = T extends 6 | | OpenAPIV3.ReferenceObject 7 | | OpenAPIV2.ReferenceObject 8 | ? never 9 | : T; 10 | 11 | export function $deref( 12 | obj: O | OpenAPIV3.ReferenceObject | OpenAPIV2.ReferenceObject, 13 | ) { 14 | return obj as ExcludeReference; 15 | } 16 | 17 | export type OperationObject = 18 | | OpenAPIV2.OperationObject 19 | | OpenAPIV3.OperationObject 20 | | OpenAPIV3_1.OperationObject; 21 | 22 | export function tokenizedLength(property: JsonValue | object) { 23 | if (typeof property === 'boolean') return 1; 24 | 25 | const output = JSON.stringify(property); 26 | return encode(output).length; 27 | } 28 | -------------------------------------------------------------------------------- /src/utils/openapi/required-schema.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema7 } from 'json-schema'; 2 | import cloneDeep from 'lodash/cloneDeep'; 3 | 4 | function extractRequiredSchema($schema: JSONSchema7 | undefined) { 5 | if (!$schema) return $schema; 6 | 7 | const schema = cloneDeep($schema); 8 | 9 | if (schema.type === 'object' && schema.properties) { 10 | const required = schema.required ?? []; 11 | 12 | for (const property in schema.properties) { 13 | if (!required.includes(property)) { 14 | delete schema.properties[property]; 15 | } 16 | } 17 | } else if (schema.type === 'array' && typeof schema.items === 'object') { 18 | if (!Array.isArray(schema.items)) { 19 | schema.items = extractRequiredSchema(schema.items); 20 | } else { 21 | for (let i = 0; i < schema.items.length; i++) { 22 | const itemSchema = schema.items[i]; 23 | if (typeof itemSchema === 'object') { 24 | schema.items[i] = extractRequiredSchema(itemSchema)!; 25 | } 26 | } 27 | } 28 | } 29 | 30 | return schema; 31 | } 32 | 33 | export { extractRequiredSchema }; 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) me@sudhanshug.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/llm/index.ts: -------------------------------------------------------------------------------- 1 | import { SomeZodObject } from 'zod'; 2 | 3 | import Logger from '../utils/logger'; 4 | 5 | export enum LLMs { 6 | OpenAI = 'openai', 7 | } 8 | 9 | export interface IChatCompletionMessage { 10 | role: 'user' | 'assistant' | 'system'; 11 | content: string; 12 | } 13 | 14 | export interface IChatCompletionResponse { 15 | content: string; 16 | usage?: any; 17 | } 18 | 19 | export enum ChatCompletionModels { 20 | trivial, 21 | critical, 22 | } 23 | 24 | export type GenerateStructuredChatCompletion< 25 | $O extends SomeZodObject = SomeZodObject, 26 | > = ( 27 | params: { 28 | model?: ChatCompletionModels; 29 | messages: IChatCompletionMessage[]; 30 | generatorName: string; 31 | generatorDescription: string; 32 | generatorOutputSchema: O; 33 | }, 34 | extraArgs?: object, 35 | ) => Promise>; 36 | 37 | export type GenerateChatCompletion = ( 38 | params: { 39 | model?: ChatCompletionModels; 40 | messages: IChatCompletionMessage[]; 41 | }, 42 | extraArgs?: object, 43 | ) => Promise; 44 | 45 | export abstract class LLM { 46 | abstract client: ClientT; 47 | abstract defaultModel: ChatCompletionModels; 48 | abstract logger: Logger; 49 | 50 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 51 | constructor(_logger: Logger) {} 52 | 53 | abstract generateStructured: GenerateStructuredChatCompletion; 54 | 55 | abstract generate: GenerateChatCompletion; 56 | } 57 | -------------------------------------------------------------------------------- /src/utils/openapi/tokenizer.ts: -------------------------------------------------------------------------------- 1 | import SwaggerParser from '@apidevtools/swagger-parser'; 2 | import { OpenAPI, OpenAPIV2 } from 'openapi-types'; 3 | 4 | import { OperationPath } from '../../agent/external-resource-directory'; 5 | 6 | import { getOauthSecuritySchemeName } from './operation'; 7 | import { OperationTokenizer } from './operation-tokenizer'; 8 | 9 | export type TokenMap = Map< 10 | string, 11 | { 12 | tokens: string; 13 | paths: Set; 14 | scopes: Set; 15 | apiSpecId: string; 16 | } 17 | >; 18 | 19 | export class OpenAPITokenizer { 20 | tokenMap: TokenMap = new Map(); 21 | 22 | constructor( 23 | private apiSpecId: string, 24 | private openapi: OpenAPI.Document, 25 | ) {} 26 | 27 | async tokenize() { 28 | this.openapi = await SwaggerParser.dereference(this.openapi); 29 | 30 | for (const path in this.paths) { 31 | const parameters = this.paths[path]!.parameters ?? []; 32 | 33 | for (const method of Object.values(OpenAPIV2.HttpMethods)) { 34 | const operation = this.paths[path]![method as OpenAPIV2.HttpMethods]; 35 | if (!operation) continue; 36 | 37 | const tokenizer = new OperationTokenizer( 38 | this.tokenMap, 39 | this.apiSpecId, 40 | path, 41 | method, 42 | operation, 43 | parameters, 44 | this.oauthSecuritySchemeName, 45 | ); 46 | tokenizer.tokenize(); 47 | } 48 | } 49 | 50 | return this.tokenMap; 51 | } 52 | 53 | get paths() { 54 | return this.openapi.paths ?? {}; 55 | } 56 | 57 | get oauthSecuritySchemeName() { 58 | return getOauthSecuritySchemeName(this.openapi); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/embeddings/index.ts: -------------------------------------------------------------------------------- 1 | import { BasicValueQuery } from 'sift'; 2 | import { Simplify } from 'type-fest'; 3 | 4 | export type InterveneParserItemMetadata = { 5 | tokens: string; 6 | apiSpecId: string; 7 | paths: string; 8 | } & Record; 9 | 10 | export type InterveneParserItemMetadataWhere = Exclude< 11 | BasicValueQuery>, 12 | RegExp 13 | >; 14 | 15 | export type InterveneParserItem = { 16 | id: string; 17 | metadata: InterveneParserItemMetadata; 18 | distance?: number; 19 | embeddings?: number[]; 20 | }; 21 | 22 | export type StorableInterveneParserItem = { 23 | id: string; 24 | metadata: InterveneParserItemMetadata; 25 | embeddings: number[]; 26 | metadataHash: string; 27 | }; 28 | 29 | export type CreateEmbeddingResponse = [string, number[]][]; 30 | export type CreateEmbeddingsFunction = ( 31 | input: string[], 32 | ) => Promise>; 33 | 34 | export type SearchEmbeddingsFunction = ( 35 | input: string, 36 | embedding: number[], 37 | limit: number, 38 | where: InterveneParserItemMetadataWhere, 39 | ) => Promise; 40 | 41 | export type UpsertEmbeddingsFunction = ( 42 | embeddings: StorableInterveneParserItem[], 43 | ) => Promise; 44 | 45 | export type RetrieveEmbeddingsFunction = ( 46 | input: string[], 47 | ) => Promise; 48 | 49 | export type VectorStoreFunctions = { 50 | searchItems: SearchEmbeddingsFunction; 51 | retrieveItems: RetrieveEmbeddingsFunction; 52 | upsertItems: UpsertEmbeddingsFunction; 53 | }; 54 | 55 | export type EmbeddingFunctions = { 56 | createEmbeddings: CreateEmbeddingsFunction; 57 | } & VectorStoreFunctions; 58 | -------------------------------------------------------------------------------- /src/utils/openapi/merge-schema.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema7 } from 'json-schema'; 2 | import cloneDeep from 'lodash/cloneDeep'; 3 | import compact from 'lodash/compact'; 4 | import flatten from 'lodash/flatten'; 5 | 6 | function mergeSchema(schema1: JSONSchema7, schema2: JSONSchema7): JSONSchema7 { 7 | const type = schema1.type ?? schema2.type; 8 | schema1.type = type; 9 | schema2.type = type; 10 | 11 | let mergedSchema = cloneDeep(schema1); 12 | 13 | if (schema1.type === 'object') { 14 | mergedSchema.required = Array.from( 15 | new Set(flatten(compact([mergedSchema.required, schema2.required]))), 16 | ); 17 | mergedSchema.properties = { 18 | ...mergedSchema.properties, 19 | ...schema2.properties, 20 | }; 21 | } else if ( 22 | mergedSchema.type === 'array' && 23 | typeof schema2.items === 'object' && 24 | !Array.isArray(schema2.items) 25 | ) { 26 | if (!schema1.items) { 27 | mergedSchema.items = schema2.items; 28 | } else if ( 29 | !Array.isArray(mergedSchema.items) && 30 | typeof mergedSchema.items === 'object' 31 | ) { 32 | mergedSchema.items = mergeSchema(mergedSchema.items, schema2.items); 33 | } else { 34 | if (Array.isArray(mergedSchema.items) && Array.isArray(schema2.items)) { 35 | for ( 36 | let i = 0; 37 | i < Math.max(mergedSchema.items.length, schema2.items.length); 38 | i += 1 39 | ) { 40 | mergedSchema.items[i] = mergeSchema( 41 | mergedSchema.items[i] as JSONSchema7, 42 | schema2.items[i], 43 | ); 44 | } 45 | } 46 | } 47 | } else { 48 | mergedSchema = { 49 | ...mergedSchema, 50 | ...schema2, 51 | }; 52 | } 53 | 54 | return mergedSchema; 55 | } 56 | 57 | export { mergeSchema }; 58 | -------------------------------------------------------------------------------- /src/utils/openapi/dereference-path.ts: -------------------------------------------------------------------------------- 1 | import SwaggerParser from '@apidevtools/swagger-parser'; 2 | import cloneDeep from 'lodash/cloneDeep'; 3 | import { OpenAPI, OpenAPIV2 } from 'openapi-types'; 4 | 5 | import { OperationObject } from './'; 6 | 7 | type J = Record | any[] | any; 8 | 9 | const flatten = ($data: J, refs: SwaggerParser.$Refs, maxDepth: number) => { 10 | function flattenRefs($data: J, refs: SwaggerParser.$Refs, d: number): J { 11 | if (d > maxDepth) { 12 | return {}; 13 | } 14 | 15 | let data = cloneDeep($data); 16 | 17 | if (data && typeof data === 'object') { 18 | if ('$ref' in data) { 19 | const ref = data['$ref'] as string; 20 | data = flattenRefs(refs.get(ref), refs, d + 1); 21 | } else { 22 | for (const key in data) { 23 | data[key] = flattenRefs(data[key], refs, d + 1); 24 | } 25 | } 26 | } 27 | 28 | return data; 29 | } 30 | return flattenRefs($data, refs, 0); 31 | }; 32 | 33 | export async function dereferencePath( 34 | jsonData: OpenAPI.Document, 35 | httpMethod: OpenAPIV2.HttpMethods, 36 | endpointPath: string, 37 | maxDepth = 13, 38 | ) { 39 | const refs = await SwaggerParser.resolve(jsonData); 40 | const api = await SwaggerParser.parse(jsonData); 41 | 42 | const operationObject = cloneDeep(api.paths?.[endpointPath]?.[httpMethod]); 43 | 44 | if (!operationObject) return {}; 45 | 46 | return flatten(operationObject, refs, maxDepth) as OperationObject; 47 | } 48 | 49 | // async function main() { 50 | // const json = await Bun.file( 51 | // '/Users/sudhanshugautam/workspace/intevene/src/server/openapi/stripe.json', 52 | // ).json(); 53 | 54 | // const o = JSON.stringify( 55 | // await dereferencePath( 56 | // json, 57 | // OpenAPIV2.HttpMethods.POST, 58 | // '/v1/subscriptions/{subscription_exposed_id}', 59 | // ), 60 | // ); 61 | 62 | // await Bun.write('/tmp/output.json', o); 63 | // } 64 | 65 | // main().then(() => { 66 | // process.exit(0); 67 | // }); 68 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.1.12", 3 | "name": "@intervene/parser", 4 | "main": "lib/main/index.js", 5 | "typings": "lib/main/index.d.ts", 6 | "module": "lib/module/index.js", 7 | "files": [ 8 | "lib/main", 9 | "lib/module", 10 | "!**/*.spec.*", 11 | "!**/*.json", 12 | "CHANGELOG.md", 13 | "LICENSE", 14 | "README.md" 15 | ], 16 | "repository": { 17 | "type": "git", 18 | "url": "https://github.com/intervenehq/parser" 19 | }, 20 | "ava": { 21 | "failFast": true, 22 | "timeout": "60s", 23 | "typescript": { 24 | "rewritePaths": { 25 | "src/": "lib/main/" 26 | }, 27 | "compile": false 28 | }, 29 | "files": [ 30 | "!lib/module/**" 31 | ] 32 | }, 33 | "scripts": { 34 | "build": "npm run build:main && npm run build:module", 35 | "build:main": "tsc -p tsconfig.json", 36 | "build:module": "tsc -p tsconfig.module.json", 37 | "dev": "tsc -p tsconfig.json -w" 38 | }, 39 | "author": { 40 | "name": "Sudhanshu Gautam", 41 | "email": "me@sudhanshug.com" 42 | }, 43 | "publishConfig": { 44 | "access": "public" 45 | }, 46 | "dependencies": { 47 | "@apidevtools/swagger-parser": "^10.1.0", 48 | "gpt-tokenizer": "^2.1.2", 49 | "handlebars": "^4.7.8", 50 | "lodash": "^4.17.21", 51 | "object-hash": "^3.0.0", 52 | "ora": "^7.0.1", 53 | "sift": "^17.0.1", 54 | "string-strip-html": "^13.4.3", 55 | "tsx": "^3.13.0", 56 | "zod": "^3.22.4" 57 | }, 58 | "devDependencies": { 59 | "@ianvs/prettier-plugin-sort-imports": "^4.1.0", 60 | "@types/json-schema": "^7.0.13", 61 | "@types/lodash": "^4.14.199", 62 | "@types/node": "^20.8.10", 63 | "@types/object-hash": "^3.0.4", 64 | "@typescript-eslint/eslint-plugin": "^6.7.5", 65 | "@typescript-eslint/parser": "^6.7.5", 66 | "bun-types": "^1.0.11", 67 | "eslint": "^8.51.0", 68 | "eslint-config-prettier": "^9.0.0", 69 | "eslint-plugin-prettier": "^5.0.0", 70 | "openapi-types": "^12.1.3", 71 | "prettier": "^3.0.3", 72 | "type-fest": "^4.3.3", 73 | "typescript": "^5.2.2" 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /tsconfig.base.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "incremental": true, 4 | "outDir": "lib/main", 5 | "moduleResolution": "node", 6 | "declaration": true, 7 | "skipLibCheck": true, 8 | "inlineSourceMap": true, 9 | "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, 10 | "resolveJsonModule": true /* Include modules imported with .json extension. */, 11 | 12 | "strict": true /* Enable all strict type-checking options. */, 13 | 14 | /* Strict Type-Checking Options */ 15 | // "noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */, 16 | // "strictNullChecks": true /* Enable strict null checks. */, 17 | // "strictFunctionTypes": true /* Enable strict checking of function types. */, 18 | // "strictPropertyInitialization": true /* Enable strict checking of property initialization in classes. */, 19 | // "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */, 20 | // "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, 21 | 22 | /* Additional Checks */ 23 | "noUnusedLocals": true /* Report errors on unused locals. */, 24 | "noUnusedParameters": true /* Report errors on unused parameters. */, 25 | "noImplicitReturns": true /* Report error when not all code paths in function return a value. */, 26 | "noFallthroughCasesInSwitch": true /* Report errors for fallthrough cases in switch statement. */, 27 | 28 | /* Debugging Options */ 29 | "traceResolution": false /* Report module resolution log messages. */, 30 | "listEmittedFiles": false /* Print names of generated files part of the compilation. */, 31 | "listFiles": false /* Print names of files part of the compilation. */, 32 | "pretty": true /* Stylize errors and messages using color and context. */, 33 | 34 | /* Experimental Options */ 35 | // "experimentalDecorators": true /* Enables experimental support for ES7 decorators. */, 36 | // "emitDecoratorMetadata": true /* Enables experimental support for emitting type metadata for decorators. */, 37 | 38 | "lib": ["es2017"], 39 | "types": [ "bun-types"] 40 | }, 41 | "include": [ 42 | "src/**/*", 43 | ], 44 | "exclude": ["node_modules/**"], 45 | "compileOnSave": false 46 | } 47 | -------------------------------------------------------------------------------- /tests/openapi/operation-tokenizer.test.ts: -------------------------------------------------------------------------------- 1 | import crypto from 'crypto'; 2 | import SwaggerParser from '@apidevtools/swagger-parser'; 3 | import { describe, expect, test } from 'bun:test'; 4 | 5 | import { OperationPath } from '../../src/agent/external-resource-directory'; 6 | import { OperationTokenizer } from '../../src/utils/openapi/operation-tokenizer'; 7 | import { TokenMap } from '../../src/utils/openapi/tokenizer'; 8 | 9 | describe('Operation Tokenizer', () => { 10 | test('should tokenize an operation', async () => { 11 | const openapi = await SwaggerParser.parse( 12 | import.meta.dir + '/../__specs__/test_v3.json', 13 | ); 14 | const putOperation = openapi.paths!['/pets']!.put!; 15 | const postOperation = openapi.paths!['/pets']!.post!; 16 | const pathItemParameters = openapi.paths!['/pets']!.parameters ?? []; 17 | 18 | const apiSpecId = 'api-spec-id'; 19 | const p1 = `${apiSpecId}:`; 20 | const p2 = `${apiSpecId}|`; 21 | function h(str: string): string { 22 | return p2 + crypto.createHash('sha256').update(str).digest('hex'); 23 | } 24 | 25 | const tokenMap: TokenMap = new Map(); 26 | 27 | new OperationTokenizer( 28 | tokenMap, 29 | apiSpecId, 30 | '/pets', 31 | 'put', 32 | putOperation, 33 | pathItemParameters, 34 | 'oa', 35 | ).tokenize(); 36 | 37 | expect(tokenMap.size).toEqual(4); 38 | 39 | const params1Id = h('param1 description'); 40 | const params2Id = h('param2 description'); 41 | const putPetsId = h('Put pets endpoint description'); 42 | 43 | expect(tokenMap.has(params1Id)).toBeTrue(); 44 | expect(tokenMap.has(params2Id)).toBeTrue(); 45 | expect(tokenMap.has(putPetsId)).toBeTrue(); 46 | 47 | expect(tokenMap.get(params1Id)!.paths).toEqual( 48 | new Set([`${p2}/pets|put`]) as Set, 49 | ); 50 | expect(tokenMap.get(params2Id)!.paths).toEqual( 51 | new Set([`${p2}/pets|put`]) as Set, 52 | ); 53 | expect(tokenMap.get(params2Id)!.scopes).toEqual( 54 | new Set([`${p1}write:pets`, `${p1}read:pets`, `${p1}admin:pets`]), 55 | ); 56 | 57 | new OperationTokenizer( 58 | tokenMap, 59 | apiSpecId, 60 | '/pets', 61 | 'post', 62 | postOperation, 63 | pathItemParameters, 64 | 'oa', 65 | ).tokenize(); 66 | 67 | expect(tokenMap.get(params2Id)!.paths).toEqual( 68 | new Set([`${p2}/pets|put`, `${p2}/pets|post`]) as Set, 69 | ); 70 | expect(tokenMap.get(params2Id)!.scopes).toEqual( 71 | new Set([ 72 | `${p1}write:pets`, 73 | `${p1}read:pets`, 74 | `${p1}admin:pets`, 75 | `${p1}create:pets`, 76 | ]), 77 | ); 78 | }); 79 | }); 80 | -------------------------------------------------------------------------------- /src/utils/openapi/deepen-schema.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema7 } from 'json-schema'; 2 | import cloneDeep from 'lodash/cloneDeep'; 3 | import compact from 'lodash/compact'; 4 | import flatten from 'lodash/flatten'; 5 | import omit from 'lodash/omit'; 6 | 7 | import { tokenizedLength } from './'; 8 | 9 | function deepenSchema( 10 | fullSchema: JSONSchema7, 11 | filteredShallowSchema: JSONSchema7, 12 | ) { 13 | if (fullSchema.type !== filteredShallowSchema.type) 14 | throw `fullSchema and filteredShallowSchema need to be of the same type, got ${JSON.stringify( 15 | fullSchema.type, 16 | )} and ${JSON.stringify(filteredShallowSchema.type)}`; 17 | 18 | let deepenedSchema = cloneDeep(filteredShallowSchema); 19 | 20 | if ( 21 | typeof fullSchema.items === 'object' && 22 | typeof filteredShallowSchema.items === 'object' 23 | ) { 24 | if ( 25 | fullSchema.type === 'array' && 26 | !Array.isArray(fullSchema.items) && 27 | !Array.isArray(filteredShallowSchema.items) 28 | ) { 29 | deepenedSchema.items = deepenSchema( 30 | fullSchema.items, 31 | filteredShallowSchema.items, 32 | ); 33 | } 34 | 35 | if (fullSchema.type === 'object') { 36 | deepenedSchema.required = Array.from( 37 | new Set( 38 | flatten( 39 | compact([fullSchema.required, filteredShallowSchema.required]), 40 | ), 41 | ), 42 | ); 43 | deepenedSchema.properties = { 44 | ...fullSchema.properties, 45 | ...filteredShallowSchema.properties, 46 | }; 47 | } 48 | } else { 49 | deepenedSchema = { 50 | ...deepenedSchema, 51 | ...filteredShallowSchema, 52 | }; 53 | } 54 | 55 | return deepenedSchema; 56 | } 57 | 58 | function shallowSchema(schema: JSONSchema7 | null | undefined) { 59 | if (!schema) return {}; 60 | 61 | const newSchema = cloneDeep(schema); 62 | 63 | const omitableProperties: (keyof JSONSchema7)[] = [ 64 | 'additionalProperties', 65 | 'patternProperties', 66 | 'definitions', 67 | 'dependencies', 68 | 'allOf', 69 | 'anyOf', 70 | 'oneOf', 71 | 'not', 72 | 'if', 73 | 'then', 74 | 'else', 75 | ]; 76 | 77 | delete newSchema.properties; 78 | if (typeof newSchema.items === 'object' && !Array.isArray(newSchema.items)) { 79 | delete newSchema.items; 80 | } else { 81 | omitableProperties.push('items'); 82 | } 83 | 84 | for (const property of omitableProperties) { 85 | // keep JSONschema validation keywords that are not too long 86 | if (tokenizedLength(newSchema[property] ?? {}) > 100) { 87 | delete newSchema[property]; 88 | } 89 | } 90 | 91 | return omit(schema, ['items']); 92 | } 93 | 94 | export { deepenSchema, shallowSchema }; 95 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore 2 | 3 | # Logs 4 | 5 | logs 6 | _.log 7 | npm-debug.log_ 8 | yarn-debug.log* 9 | yarn-error.log* 10 | lerna-debug.log* 11 | .pnpm-debug.log* 12 | 13 | # Diagnostic reports (https://nodejs.org/api/report.html) 14 | 15 | report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json 16 | 17 | # Runtime data 18 | 19 | pids 20 | _.pid 21 | _.seed 22 | \*.pid.lock 23 | 24 | # Directory for instrumented libs generated by jscoverage/JSCover 25 | 26 | lib-cov 27 | 28 | # Coverage directory used by tools like istanbul 29 | 30 | coverage 31 | \*.lcov 32 | 33 | # nyc test coverage 34 | 35 | .nyc_output 36 | 37 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 38 | 39 | .grunt 40 | 41 | # Bower dependency directory (https://bower.io/) 42 | 43 | bower_components 44 | 45 | # node-waf configuration 46 | 47 | .lock-wscript 48 | 49 | # Compiled binary addons (https://nodejs.org/api/addons.html) 50 | 51 | build/Release 52 | 53 | # Dependency directories 54 | 55 | node_modules/ 56 | jspm_packages/ 57 | 58 | # Snowpack dependency directory (https://snowpack.dev/) 59 | 60 | web_modules/ 61 | 62 | # TypeScript cache 63 | 64 | \*.tsbuildinfo 65 | 66 | # Optional npm cache directory 67 | 68 | .npm 69 | 70 | # Optional eslint cache 71 | 72 | .eslintcache 73 | 74 | # Optional stylelint cache 75 | 76 | .stylelintcache 77 | 78 | # Microbundle cache 79 | 80 | .rpt2_cache/ 81 | .rts2_cache_cjs/ 82 | .rts2_cache_es/ 83 | .rts2_cache_umd/ 84 | 85 | # Optional REPL history 86 | 87 | .node_repl_history 88 | 89 | # Output of 'npm pack' 90 | 91 | \*.tgz 92 | 93 | # Yarn Integrity file 94 | 95 | .yarn-integrity 96 | 97 | # dotenv environment variable files 98 | 99 | .env 100 | .env.development.local 101 | .env.test.local 102 | .env.production.local 103 | .env.local 104 | 105 | # parcel-bundler cache (https://parceljs.org/) 106 | 107 | .cache 108 | .parcel-cache 109 | 110 | # Next.js build output 111 | 112 | .next 113 | out 114 | 115 | # Nuxt.js build / generate output 116 | 117 | .nuxt 118 | dist 119 | lib 120 | 121 | # Gatsby files 122 | 123 | .cache/ 124 | 125 | # Comment in the public line in if your project uses Gatsby and not Next.js 126 | 127 | # https://nextjs.org/blog/next-9-1#public-directory-support 128 | 129 | # public 130 | 131 | # vuepress build output 132 | 133 | .vuepress/dist 134 | 135 | # vuepress v2.x temp and cache directory 136 | 137 | .temp 138 | .cache 139 | 140 | # Docusaurus cache and generated files 141 | 142 | .docusaurus 143 | 144 | # Serverless directories 145 | 146 | .serverless/ 147 | 148 | # FuseBox cache 149 | 150 | .fusebox/ 151 | 152 | # DynamoDB Local files 153 | 154 | .dynamodb/ 155 | 156 | # TernJS port file 157 | 158 | .tern-port 159 | 160 | # Stores VSCode versions used for testing VSCode extensions 161 | 162 | .vscode-test 163 | 164 | # yarn v2 165 | 166 | .yarn/cache 167 | .yarn/unplugged 168 | .yarn/build-state.yml 169 | .yarn/install-state.gz 170 | .pnp.\* 171 | 172 | # IntelliJ based IDEs 173 | .idea 174 | 175 | .tmp 176 | 177 | .DS_Store 178 | chroma.log 179 | 180 | src/playground 181 | output.json 182 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Intervene Parser 2 | 3 |

4 | Intervene Parser Logo 5 |
6 |
7 | 8 | Discord 9 | 10 | 11 | [Introduction](#introduction) | 12 | [How to use](#usage) | 13 | [FAQs](#faqs) | 14 | [Contact](mailto:me@sudhanshug.com) 15 | 16 |

17 | 18 | > Launching a drop-in replacement to Zapier NLA by Nov 7th! Please contact me [me@sudhanshug.com](mailto:me@sudhanshug.com) if you have any questions. 19 | 20 | ## Introduction 21 | 22 | _Open source Natural Language Actions (NLA)_ 23 | 24 | Translate natural language into API calls. 25 | 26 |
27 | 28 | ### Here is a web based demo to try it out: 29 | https://tryintervene.github.io/parser-demo/ 30 | 31 |
32 | 33 | Here's a quick demo video: 34 | 35 | https://github.com/tryintervene/parser/assets/9914480/b91eb0d2-64c5-4231-8989-1c27a105c3be 36 | 37 | Here's a sample output: 38 | 39 | ```bash 40 | { 41 | "provider": "", 42 | "method": "", 43 | "path": "", 44 | "bodyParams": "", 45 | "queryParams": "", 46 | "pathParams": "", 47 | "requestContentType": "application/x-www-form-urlencoded", 48 | "responseContentType": "application/json", 49 | "responseSchema": "" 50 | } 51 | ``` 52 | 53 | ## Usage 54 | 55 | You can install the parser by running: 56 | 57 | ```bash 58 | npm install @intervene/parser 59 | ``` 60 | 61 | > Note: The project is under active development and has not reached v0 yet. Proceed with caution and report any issues you may notice. 62 | 63 | ## FAQs 64 | 65 | ### This looks cool, but what about prod? 66 | 67 | You can use the library as is in production but proceed with caution as it is under active development. 68 | 69 | If you're interested in a hosted solution, please [fill out this quick form](https://tally.so/r/wzMJ8a), and I will get back to you in no time! 70 | 71 | ### Can it run with GPT 3.5? 72 | 73 | You can use GPT 3.5 (or equivalent) which will make this a lot faster, cheaper but less accurate. You can go this route for simpler API calls that need to extract data from the user prompt. You can use the `--trivial` flag to do this 74 | 75 | However, the code can be optimized to use the less capable models for selective tasks. Open to PRs :) 76 | 77 | ### What about other LLMs? 78 | 79 | This project works only with OpenAI models for now. I will be exploring other LLMs as well. Let me know [which one you want by opening an issue here](https://github.com/tryintervene/parser/issues/new?title=Request%20to%20integrate%20LLM:%20[LLM]&body=Hi,%20can%20you%20please%20add%20the%20following%20LLM%20to%20the%20parser:%20) or feel free to open a PR! 80 | 81 | ### Umm I don't like JavaScript. What about python? 82 | 83 | Before porting it to Python or Golang (or both), I want to determine if there are any real-world use cases for this technology. Please try out the CLI, share your thoughts, and I will promptly port it to other languages based on the feedback. 84 | 85 | I chose to start with a statically typed language due to the nature of the project. I could have used Golang, but I aimed for simplicity, hence the choice of TypeScript. 86 | 87 | ### I want to contribute 88 | 89 | Awesome! PRs and issues are welcome! 90 | 91 | ## Credits 92 | 93 | Credits to [LangChain](https://github.com/langchain-ai/langchain) and [LlamaIndex](https://github.com/run-llama/llama_index) for the inspiration for some of the techniques used in the project. 94 | 95 | Special credits to [@rohanmayya](https://github.com/rohanmayya) for helping lay the foundation for this project. 96 | -------------------------------------------------------------------------------- /src/utils/openapi/chunk-schema.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema7 } from 'json-schema'; 2 | import cloneDeep from 'lodash/cloneDeep'; 3 | 4 | import { tokenizedLength } from '.'; 5 | import { shallowSchema } from './deepen-schema'; 6 | 7 | function chunkSchema( 8 | schema: JSONSchema7, 9 | chunkRequiredProperties = false, 10 | tokenLimit = 4000, 11 | ) { 12 | if ( 13 | !schema.type || 14 | Array.isArray(schema.type) || 15 | !['array', 'object'].includes(schema.type) 16 | ) { 17 | return [{ propertyNames: [], schema }]; 18 | } 19 | 20 | const metadata = shallowSchema(schema); 21 | 22 | const chunkedSchema: { 23 | propertyNames: string[]; 24 | schema: JSONSchema7; 25 | }[] = []; 26 | 27 | if (schema.type === 'object') { 28 | const { chunks } = chunkProperties( 29 | schema, 30 | chunkRequiredProperties, 31 | tokenLimit, 32 | ); 33 | 34 | chunks.map((chunk) => { 35 | chunkedSchema.push({ 36 | propertyNames: Object.keys(chunk), 37 | schema: { ...metadata, properties: chunk }, 38 | }); 39 | }); 40 | } else if (!Array.isArray(schema.items) && typeof schema.items === 'object') { 41 | const { chunks } = chunkProperties( 42 | schema.items, 43 | chunkRequiredProperties, 44 | tokenLimit, 45 | ); 46 | 47 | const itemMetadata = shallowSchema(schema.items); 48 | 49 | chunks.map((chunk) => { 50 | chunkedSchema.push({ 51 | propertyNames: Object.keys(chunk), 52 | schema: { 53 | ...metadata, 54 | items: { 55 | ...itemMetadata, 56 | properties: chunk, 57 | }, 58 | }, 59 | }); 60 | }); 61 | } 62 | 63 | return chunkedSchema; 64 | } 65 | 66 | function chunkProperties( 67 | objectSchema: JSONSchema7, 68 | chunkRequiredProperties = false, 69 | tokenLimit = 4000, 70 | ): { 71 | required: JSONSchema7['properties']; 72 | chunks: NonNullable[]; 73 | } { 74 | if (objectSchema.type !== 'object') 75 | return { required: objectSchema.properties, chunks: [] }; 76 | 77 | const chunks: NonNullable[] = []; 78 | let requiredProperties: JSONSchema7['properties']; 79 | const { properties, required } = objectSchema; 80 | 81 | let currLength = 0; 82 | let curr: JSONSchema7['properties'] | undefined; 83 | 84 | for (const propertyName in properties) { 85 | const property = properties[propertyName]!; 86 | 87 | if (typeof property === 'boolean') continue; 88 | 89 | const propertyMetadata = shallowSchema(property); 90 | if (!chunkRequiredProperties && !!required?.includes(propertyName)) { 91 | requiredProperties ||= {}; 92 | requiredProperties[propertyName] = propertyMetadata; 93 | continue; 94 | } 95 | 96 | const length = tokenizedLength(propertyMetadata); 97 | 98 | if (currLength + length > tokenLimit || length > tokenLimit) { 99 | chunks.push(curr!); 100 | curr = undefined; 101 | currLength = 0; 102 | } 103 | 104 | curr ||= {}; 105 | 106 | curr[propertyName] = propertyMetadata; 107 | currLength += length; 108 | } 109 | 110 | if (currLength > 0) { 111 | chunks.push(curr!); 112 | } 113 | 114 | return { required: requiredProperties, chunks }; 115 | } 116 | 117 | function getSubSchema(schema: JSONSchema7, properties: string[]) { 118 | const subSchema: JSONSchema7 = cloneDeep(schema); 119 | 120 | if (schema.type === 'object') { 121 | subSchema.properties = {}; 122 | 123 | for (const propertyName of properties) { 124 | subSchema.properties[propertyName] = schema.properties![propertyName]!; 125 | } 126 | } else if ( 127 | schema.type === 'array' && 128 | typeof schema.items === 'object' && 129 | !Array.isArray(schema.items) 130 | ) { 131 | subSchema.items = getSubSchema(schema.items, properties); 132 | } 133 | 134 | return subSchema; 135 | } 136 | 137 | export { chunkSchema, getSubSchema }; 138 | -------------------------------------------------------------------------------- /src/utils/openapi/operation-tokenizer.ts: -------------------------------------------------------------------------------- 1 | import crypto from 'crypto'; 2 | import { encode } from 'gpt-tokenizer'; 3 | import { IJsonSchema, OpenAPI } from 'openapi-types'; 4 | import { stripHtml } from 'string-strip-html'; 5 | 6 | import { OperationPath } from '../../agent/external-resource-directory'; 7 | 8 | import { $deref } from '.'; 9 | import { getDefaultContentType } from './content-type'; 10 | import { getOperationScopes } from './operation'; 11 | import { TokenMap } from './tokenizer'; 12 | 13 | export class OperationTokenizer { 14 | constructor( 15 | private tokenMap: TokenMap, 16 | private apiSpecId: string, 17 | private urlPath: string, 18 | private httpMethod: string, 19 | private operationObj: OpenAPI.Operation, 20 | private pathItemParameters: OpenAPI.Parameter[], 21 | private oauthSecuritySchemeName: string | undefined, 22 | ) {} 23 | 24 | tokenize() { 25 | this.addOperationDescription(); 26 | this.addParameters(); 27 | this.addRequestBody(); 28 | } 29 | 30 | private addOperationDescription() { 31 | const operationDescription = 32 | this.operationObj.description ?? 33 | this.operationObj.operationId ?? 34 | this.operationObj.summary ?? 35 | ''; 36 | this.addEntry(operationDescription); 37 | } 38 | 39 | private addParameters() { 40 | const parameters = [ 41 | ...this.pathItemParameters, 42 | ...(this.operationObj.parameters ?? []), 43 | ]; 44 | for (const $parameter of parameters) { 45 | const parameter = $deref($parameter); 46 | 47 | const parameterDescription = 48 | parameter.description ?? parameter.name ?? ''; 49 | this.addEntry(parameterDescription); 50 | } 51 | } 52 | 53 | private addRequestBody() { 54 | if ( 55 | !('requestBody' in this.operationObj && !!this.operationObj.requestBody) 56 | ) { 57 | return; 58 | } 59 | const requestBody = $deref(this.operationObj.requestBody); 60 | 61 | const defaultContentType = getDefaultContentType( 62 | Object.keys(requestBody.content), 63 | ); 64 | const mediaTypeObject = requestBody.content[defaultContentType]; 65 | const schema = $deref(mediaTypeObject.schema); 66 | if (!schema) return; 67 | 68 | let properties: Record; 69 | 70 | if (schema.type === 'object') { 71 | properties = schema.properties ?? {}; 72 | } else if (schema.type === 'array' && !Array.isArray(schema.items)) { 73 | properties = $deref(schema.items)?.properties ?? {}; 74 | } else { 75 | return; 76 | } 77 | 78 | for (const [propertyName, property] of Object.entries(properties)) { 79 | const propertySchema = property as IJsonSchema; 80 | const propertyDescription = 81 | propertySchema.description ?? propertyName ?? ''; 82 | this.addEntry(propertyDescription); 83 | } 84 | } 85 | 86 | private addEntry(tokens: string) { 87 | const id = 88 | this.apiSpecId + 89 | '|' + 90 | crypto.createHash('sha256').update(tokens).digest('hex'); 91 | 92 | // eslint-disable-next-line no-control-regex 93 | let escapedTokens = stripHtml(tokens).result.replace(/[^\x00-\x7F]/g, ''); 94 | while (encode(escapedTokens).length > 8000) { 95 | escapedTokens = escapedTokens.slice(0, -100); 96 | } 97 | 98 | if (!this.tokenMap.has(id)) { 99 | this.tokenMap.set(id, { 100 | tokens: escapedTokens, 101 | paths: new Set(), 102 | scopes: new Set(), 103 | apiSpecId: this.apiSpecId, 104 | }); 105 | } 106 | 107 | this.tokenMap.get(id)!.paths.add(this.path); 108 | this.oauthScopes.map((s) => this.tokenMap.get(id)!.scopes.add(s)); 109 | } 110 | 111 | private get oauthScopes() { 112 | return getOperationScopes( 113 | this.apiSpecId, 114 | this.operationObj, 115 | this.oauthSecuritySchemeName, 116 | ); 117 | } 118 | 119 | private get path() { 120 | return `${this.apiSpecId}|${this.urlPath}|${this.httpMethod}` as OperationPath; 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /tests/__specs__/test_v3.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://spec.openapis.org/oas/3.0/schema/latest", 3 | "info": { 4 | "title": "OpenAPI Petstore", 5 | "version": "1.0.0", 6 | "description": "This is a sample Pet Store Server based on the OpenAPI 3.0 specification" 7 | }, 8 | "openapi": "3.0.0", 9 | "paths": { 10 | "/pets": { 11 | "parameters": [ 12 | { 13 | "name": "param1", 14 | "description": "param1 description", 15 | "in": "query", 16 | "required": true, 17 | "schema": { 18 | "type": "string" 19 | } 20 | } 21 | ], 22 | "put": { 23 | "summary": "Put pets endpoint", 24 | "description": "Put pets endpoint description", 25 | "operationId": "getEndpoint", 26 | "parameters": [ 27 | { 28 | "name": "param2", 29 | "description": "param2 description", 30 | "in": "query", 31 | "required": true, 32 | "schema": { 33 | "type": "number" 34 | } 35 | } 36 | ], 37 | "requestBody": { 38 | "description": "Request body description", 39 | "content": { 40 | "application/json": { 41 | "schema": { 42 | "type": "object", 43 | "properties": { 44 | "prop1": { 45 | "description": "prop1 description", 46 | "type": "string" 47 | } 48 | } 49 | } 50 | } 51 | } 52 | }, 53 | "security": [ 54 | { 55 | "oa": ["write:pets", "read:pets"] 56 | }, 57 | { 58 | "oa": ["admin:pets"] 59 | } 60 | ], 61 | "responses": { 62 | "200": { 63 | "description": "OK", 64 | "content": { 65 | "application/json": { 66 | "schema": { 67 | "type": "object", 68 | "properties": { 69 | "prop1": { 70 | "type": "string" 71 | } 72 | } 73 | } 74 | } 75 | } 76 | } 77 | } 78 | }, 79 | "post": { 80 | "summary": "Post endpoint", 81 | "description": "Post endpoint description", 82 | "operationId": "postEndpoint", 83 | "parameters": [ 84 | { 85 | "name": "param2", 86 | "description": "param2 description", 87 | "in": "query", 88 | "required": true, 89 | "schema": { 90 | "type": "number" 91 | } 92 | } 93 | ], 94 | "requestBody": { 95 | "description": "Request body description", 96 | "content": { 97 | "application/json": { 98 | "schema": { 99 | "type": "object", 100 | "properties": { 101 | "prop1": { 102 | "description": "post prop1 description", 103 | "type": "string" 104 | } 105 | } 106 | } 107 | } 108 | } 109 | }, 110 | "security": [ 111 | { 112 | "oa": ["create:pets"] 113 | }, 114 | { 115 | "oa": ["admin:pets"] 116 | } 117 | ], 118 | "responses": { 119 | "200": { 120 | "description": "OK", 121 | "content": { 122 | "application/json": { 123 | "schema": { 124 | "type": "object", 125 | "properties": { 126 | "prop1": { 127 | "type": "string" 128 | } 129 | } 130 | } 131 | } 132 | } 133 | } 134 | } 135 | } 136 | } 137 | }, 138 | "components": { 139 | "schemas": {}, 140 | "securitySchemes": { 141 | "oa": { 142 | "type": "oauth2", 143 | "flows": { 144 | "implicit": { 145 | "authorizationUrl": "https://example.com/api/oauth/dialog", 146 | "scopes": { 147 | "write:pets": "modify pets in your account", 148 | "read:pets": "read your pets", 149 | "create:pets": "create pets", 150 | "admin:pets": "admin pets" 151 | } 152 | } 153 | } 154 | } 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/utils/openapi/operation.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema7 } from 'json-schema'; 2 | import cloneDeep from 'lodash/cloneDeep'; 3 | import { OpenAPI, OpenAPIV2, OpenAPIV3 } from 'openapi-types'; 4 | 5 | import { $deref, OperationObject } from './'; 6 | import { getDefaultContentType } from './content-type'; 7 | 8 | function extractOperationSchemas(operationObject: OperationObject) { 9 | const parameters = $deref(operationObject.parameters) ?? []; 10 | const requestBody = 11 | 'requestBody' in operationObject 12 | ? $deref(operationObject.requestBody) 13 | : undefined; 14 | const successResponse = $deref( 15 | operationObject.responses?.['200'] ?? 16 | operationObject.responses?.['201'] ?? 17 | operationObject.responses?.['204'] ?? 18 | operationObject.responses?.default, 19 | ); 20 | 21 | let querySchema: JSONSchema7 | undefined; 22 | let bodySchema: JSONSchema7 | undefined; 23 | let pathSchema: JSONSchema7 | undefined; 24 | let headerSchema: JSONSchema7 | undefined; 25 | let cookieSchema: JSONSchema7 | undefined; 26 | let responseSchema: JSONSchema7 | undefined; 27 | let requestContentType: string | undefined; 28 | let responseContentType: string | undefined; 29 | 30 | for (const $parameter of parameters) { 31 | const parameter = $deref($parameter); 32 | switch (parameter.in) { 33 | case 'query': 34 | querySchema = appendParameterToSchema(querySchema, parameter); 35 | break; 36 | case 'path': 37 | pathSchema = appendParameterToSchema(pathSchema, parameter); 38 | break; 39 | case 'header': 40 | headerSchema = appendParameterToSchema(headerSchema, parameter); 41 | break; 42 | case 'cookie': 43 | cookieSchema = appendParameterToSchema(cookieSchema, parameter); 44 | break; 45 | case 'body': 46 | bodySchema = appendParameterToSchema(bodySchema, parameter); 47 | break; 48 | } 49 | } 50 | 51 | if (requestBody) { 52 | requestContentType = getDefaultContentType( 53 | Object.keys(requestBody.content), 54 | ); 55 | bodySchema = requestBody.content[requestContentType]!.schema as JSONSchema7; 56 | } 57 | 58 | if (successResponse) { 59 | if ('schema' in successResponse) { 60 | responseSchema = successResponse.schema as JSONSchema7; 61 | } else if ('content' in successResponse) { 62 | responseContentType = getDefaultContentType( 63 | Object.keys(successResponse.content ?? {}), 64 | ); 65 | responseSchema = successResponse.content?.[responseContentType] 66 | ?.schema as JSONSchema7; 67 | } 68 | } 69 | 70 | return { 71 | querySchema, 72 | bodySchema, 73 | pathSchema, 74 | headerSchema, 75 | cookieSchema, 76 | responseSchema, 77 | requestContentType, 78 | responseContentType, 79 | }; 80 | } 81 | 82 | /** 83 | * @returns always an "object" schema 84 | */ 85 | function appendParameterToSchema( 86 | $schema: JSONSchema7 | undefined, 87 | parameter: OpenAPIV2.ParameterObject | OpenAPIV3.ParameterObject, 88 | ) { 89 | const schema = cloneDeep($schema) ?? { 90 | type: 'object', 91 | required: [], 92 | title: `${parameter.in} parameters to be sent with the HTTP request`, 93 | }; 94 | 95 | schema.properties ||= {}; 96 | 97 | schema.properties[parameter.name] = parameter.schema; 98 | if (parameter.required) { 99 | schema.required?.push(parameter.name); 100 | } 101 | 102 | return schema; 103 | } 104 | 105 | function getOperationScopes( 106 | specId: string, 107 | operationObject: OperationObject, 108 | oauthSecuritySchemeName: string | undefined, 109 | ) { 110 | if (!operationObject.security) return []; 111 | 112 | let scopes: string[] | undefined; 113 | 114 | if (oauthSecuritySchemeName) { 115 | for (const securityReq of operationObject.security) { 116 | const securityScopes = securityReq[oauthSecuritySchemeName]; 117 | if (!securityScopes) continue; 118 | 119 | scopes ||= []; 120 | scopes.push(...decorateScopes(specId, securityScopes)); 121 | } 122 | } 123 | 124 | return scopes ?? decorateScopes(specId, ['_default']); 125 | } 126 | 127 | function decorateScopes(specId: string, scope: string[]) { 128 | return scope.map((s) => `${specId}:${s}`); 129 | } 130 | 131 | function getOauthSecuritySchemeName(openapi: OpenAPI.Document) { 132 | if ('securityDefinitions' in openapi && openapi.securityDefinitions) { 133 | for (const [name, securityDefinition] of Object.entries( 134 | openapi.securityDefinitions, 135 | )) { 136 | if (securityDefinition.type === 'oauth2') { 137 | return name; 138 | } 139 | } 140 | } 141 | 142 | if ( 143 | 'components' in openapi && 144 | openapi.components && 145 | 'securitySchemes' in openapi.components && 146 | openapi.components.securitySchemes 147 | ) { 148 | for (const [name, $securityScheme] of Object.entries( 149 | openapi.components.securitySchemes, 150 | )) { 151 | const securityScheme = $deref($securityScheme); 152 | if (securityScheme.type === 'oauth2') { 153 | return name; 154 | } 155 | } 156 | } 157 | 158 | return undefined; 159 | } 160 | 161 | export { 162 | extractOperationSchemas, 163 | getOauthSecuritySchemeName, 164 | getOperationScopes, 165 | }; 166 | -------------------------------------------------------------------------------- /src/agent/code-gen.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema7 } from 'json-schema'; 2 | 3 | import { 4 | objectivePrefix, 5 | OperationMetdata, 6 | operationPrefix, 7 | } from '../agent/index'; 8 | import { IChatCompletionMessage, LLM } from '../llm'; 9 | import Logger from '../utils/logger'; 10 | import { t } from '../utils/template'; 11 | 12 | export enum CodeGenLanguage { 13 | javascript = 'javascript', 14 | python = 'python', 15 | ruby = 'ruby', 16 | php = 'php', 17 | } 18 | 19 | const functionBoilerplate = (name: string) => ({ 20 | [CodeGenLanguage.javascript]: ` 21 | function ${name}() { 22 | // any additional logic here 23 | return ; 24 | }`, 25 | 26 | [CodeGenLanguage.python]: ` 27 | def ${name}(): 28 | # any additional logic here 29 | return `, 30 | 31 | [CodeGenLanguage.ruby]: ` 32 | def ${name} 33 | # any additional logic here 34 | return 35 | end`, 36 | 37 | [CodeGenLanguage.php]: ` 38 | function ${name}() { 39 | // any additional logic here 40 | return ; 41 | }`, 42 | }); 43 | 44 | export default class CodeGenerator { 45 | constructor( 46 | public logger: Logger, 47 | public llm: LLM, 48 | public language: CodeGenLanguage, 49 | ) {} 50 | 51 | async generateInputParamCode( 52 | params: OperationMetdata & { 53 | inputSchema: JSONSchema7; 54 | context: OperationMetdata['context']; 55 | name: string; 56 | }, 57 | ) { 58 | const boilerplate = functionBoilerplate(`get_${params.name}_params`)[ 59 | this.language 60 | ]; 61 | 62 | const message = t( 63 | [ 64 | objectivePrefix(params, false), 65 | operationPrefix(params), 66 | '{{#if needsContext}}', 67 | 'Here is some data that can be used to construct the output:', 68 | '{{#each context}}`{{@key}}`: ```{{this}}```\n{{/each}}', 69 | '{{/if}}', 70 | 'Your task is to generate a function in {{language}} that follows exactly this format:', 71 | '```' + boilerplate + '```', 72 | 'Replace , with a value that satisfies the following JSON schema (call it returnSchema):', 73 | '```{{inputSchema}}```', 74 | 'Rules:', 75 | '1. You can only use data hidden in the objective and context. You must use it directly.', 76 | '2. You must not assume or imagine any piece of data.', 77 | '3. You must reply only with the function definition. No comments or explanations.', 78 | '4. The generated function must return a value that complies with the given returnSchema.', 79 | "5. You are going to reply with code that is directly eval'd on a server. Do not wrap it with markdown or '```'.", 80 | '6. The generated function MUST have EXACTLY 0 arguments.', 81 | '7. You do not have access to any external libraries or window or document objects.', 82 | '8. You can use any global variables present in a typical nodejs environment.', 83 | // '{{#if needsContext}}7. You can use any of the variables by their names directly in the gnerated code. They will be available in the context during execution. {{/if}}', 84 | ], 85 | { 86 | inputSchema: JSON.stringify(params.inputSchema), 87 | needsContext: params.context && !!Object.keys(params.context).length, 88 | context: Object.fromEntries( 89 | Object.entries(params.context || {}).map(([key, schema]) => { 90 | return [key, JSON.stringify(schema)]; 91 | }), 92 | ), 93 | language: this.language, 94 | }, 95 | ); 96 | 97 | const messages: IChatCompletionMessage[] = [ 98 | { 99 | role: 'user', 100 | content: message, 101 | }, 102 | { 103 | role: 'system', 104 | content: t([ 105 | 'This is NOT correct response:', 106 | `\`\`\`${boilerplate}\`\`\``, 107 | 'This is also NOT correct response:', 108 | `\`\`\`${this.language} ${boilerplate}\`\`\``, 109 | 'This is correct response:', 110 | boilerplate, 111 | 'IN SHORT: no markdown, no tildes', 112 | ]), 113 | }, 114 | ]; 115 | 116 | let generatedCode = await this.llm.generate( 117 | { 118 | messages, 119 | }, 120 | { 121 | logit_bias: { 122 | // GPT specific, telling GPT to not have ``` in the output (forcing it to not generate markdown) 123 | '15506': -1, 124 | }, 125 | }, 126 | ); 127 | 128 | if (generatedCode.content.match(/^[\s\n]*```(.|\n|\s)*```[\s\n]*$/)) { 129 | generatedCode = await this.llm.generate( 130 | { 131 | messages: [ 132 | ...messages, 133 | { 134 | role: 'assistant', 135 | content: generatedCode.content, 136 | }, 137 | { 138 | role: 'user', 139 | content: 140 | 'You did not follow the instructions properly. I specifically asked you not to generate markdown. Please try again.', 141 | }, 142 | ], 143 | }, 144 | { 145 | logit_bias: { 146 | '15506': -1, 147 | }, 148 | }, 149 | ); 150 | } 151 | 152 | await this.logger.info( 153 | `Generated code for ${params.name}`, 154 | generatedCode.content, 155 | ); 156 | 157 | return generatedCode.content; 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /src/agent/context-processor.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema7 } from 'json-schema'; 2 | import Zod from 'zod'; 3 | 4 | import { 5 | objectivePrefix, 6 | OperationMetdata, 7 | operationPrefix, 8 | } from '../agent/index'; 9 | import { LLM } from '../llm'; 10 | import Logger from '../utils/logger'; 11 | import { chunkSchema, getSubSchema } from '../utils/openapi/chunk-schema'; 12 | import { deepenSchema, shallowSchema } from '../utils/openapi/deepen-schema'; 13 | import { mergeSchema } from '../utils/openapi/merge-schema'; 14 | import { t } from '../utils/template'; 15 | 16 | export default class ContextProcessor { 17 | constructor( 18 | public logger: Logger, 19 | public llm: LLM, 20 | ) {} 21 | 22 | async filter( 23 | params: OperationMetdata & { 24 | inputSchema: { 25 | body: JSONSchema7; 26 | query: JSONSchema7; 27 | path: JSONSchema7; 28 | }; 29 | }, 30 | ) { 31 | const contextShortlist = ( 32 | await Promise.allSettled([ 33 | await this.shortlist({ 34 | ...params, 35 | inputSchema: params.inputSchema.body, 36 | }), 37 | await this.shortlist({ 38 | ...params, 39 | inputSchema: params.inputSchema.query, 40 | }), 41 | await this.shortlist({ 42 | ...params, 43 | inputSchema: params.inputSchema.path, 44 | }), 45 | ]) 46 | ).map((result) => { 47 | if (result.status === 'rejected') 48 | throw `couldnt shortlist context, error ${result.reason}`; 49 | 50 | return result.value; 51 | }); 52 | 53 | const filteredContextForBody = await this.filterSchema({ 54 | ...params, 55 | inputSchema: params.inputSchema.body, 56 | contextShortlist: contextShortlist[0], 57 | }); 58 | const filteredContextForQuery = await this.filterSchema({ 59 | ...params, 60 | inputSchema: params.inputSchema.query, 61 | contextShortlist: contextShortlist[1], 62 | }); 63 | const filteredContextForPath = await this.filterSchema({ 64 | ...params, 65 | inputSchema: params.inputSchema.path, 66 | contextShortlist: contextShortlist[2], 67 | }); 68 | 69 | return { 70 | filteredContextForBody, 71 | filteredContextForQuery, 72 | filteredContextForPath, 73 | }; 74 | } 75 | 76 | private async filterSchema( 77 | params: OperationMetdata & { 78 | inputSchema: JSONSchema7; 79 | contextShortlist: string[]; 80 | }, 81 | ) { 82 | const filteredContext: Record = {}; 83 | 84 | for (const key of params.contextShortlist) { 85 | let filteredSchema = shallowSchema(params.context?.[key]); 86 | const chunks = chunkSchema(params.context?.[key], true); 87 | 88 | for (const { schema: chunkSchema, propertyNames } of chunks) { 89 | if (!propertyNames.length) { 90 | filteredSchema = mergeSchema(filteredSchema, chunkSchema); 91 | continue; 92 | } 93 | 94 | const { shortlist } = await this.llm.generateStructured({ 95 | messages: [ 96 | { 97 | role: 'user', 98 | content: t( 99 | [ 100 | objectivePrefix(params, false), 101 | operationPrefix(params), 102 | 'And I came up with this input JSON schema to the resource:', 103 | '```{{inputSchema}}```', 104 | 'Here is a JSON schema of a variable named `{{key}}`:', 105 | '```{{chunkSchema}}```', 106 | "Your task is to shortlist a conservative set of properties from {{key}}'s" + 107 | ' JSON schema which may be relevant to generate an input compliant to the input schema.', 108 | ], 109 | { 110 | inputSchema: JSON.stringify(params.inputSchema), 111 | key, 112 | chunkSchema: JSON.stringify(chunkSchema), 113 | }, 114 | ), 115 | }, 116 | ], 117 | generatorName: 'shortlist_properties', 118 | generatorDescription: 119 | 'Shortlist properties that are relevant given the objective', 120 | generatorOutputSchema: Zod.object({ 121 | shortlist: Zod.array(Zod.enum(propertyNames as [string])), 122 | }), 123 | }); 124 | 125 | const subSchema = getSubSchema(chunkSchema, shortlist); 126 | filteredSchema = mergeSchema(filteredSchema, subSchema); 127 | } 128 | 129 | filteredContext[key] = deepenSchema(params.context[key], filteredSchema); 130 | } 131 | 132 | return filteredContext; 133 | } 134 | 135 | private async shortlist( 136 | params: OperationMetdata & { 137 | inputSchema: JSONSchema7; 138 | }, 139 | ) { 140 | if (!params.context || Object.keys(params.context).length === 0) { 141 | return []; 142 | } 143 | 144 | const { shortlist } = await this.llm.generateStructured({ 145 | messages: [ 146 | { 147 | role: 'user', 148 | content: t( 149 | [ 150 | objectivePrefix(params), 151 | operationPrefix(params), 152 | 'And this is the input JSON schema to the resource:', 153 | '```{{inputSchema}}```', 154 | ], 155 | { 156 | description: params.description, 157 | inputSchema: JSON.stringify(params.inputSchema), 158 | }, 159 | ), 160 | }, 161 | ], 162 | generatorName: 'shortlist_context', 163 | generatorDescription: 164 | 'Shortlist context datum that are relevant given the objective', 165 | generatorOutputSchema: Zod.object({ 166 | shortlist: Zod.array(Zod.enum(Object.keys(params.context) as [string])), 167 | }), 168 | }); 169 | 170 | return shortlist; 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/agent/external-resource-evaluator.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema7 } from 'json-schema'; 2 | import Zod from 'zod'; 3 | 4 | import { 5 | objectivePrefix, 6 | OperationMetdata, 7 | operationPrefix, 8 | } from '../agent/index'; 9 | import { ChatCompletionModels, IChatCompletionMessage, LLM } from '../llm'; 10 | import Logger from '../utils/logger'; 11 | import { chunkSchema, getSubSchema } from '../utils/openapi/chunk-schema'; 12 | import { deepenSchema } from '../utils/openapi/deepen-schema'; 13 | import { mergeSchema } from '../utils/openapi/merge-schema'; 14 | import { t } from '../utils/template'; 15 | 16 | export default class ExternalResourceEvaluator { 17 | constructor( 18 | public logger: Logger, 19 | public llm: LLM, 20 | ) {} 21 | 22 | async isFeasible( 23 | params: OperationMetdata & { 24 | requestSchema: { 25 | body?: JSONSchema7; 26 | query?: JSONSchema7; 27 | path?: JSONSchema7; 28 | }; 29 | }, 30 | ) { 31 | const message = t( 32 | [ 33 | objectivePrefix(params), 34 | 'I want to check if this API is the right choice for the task:', 35 | operationPrefix(params, true), 36 | '{{#if bodySchema}}body schema: {{bodySchema}}{{/if}}', 37 | '{{#if querySchema}}query params schema: {{querySchema}}{{/if}}', 38 | '{{#if pathSchema}}path params schema: {{pathSchema}}{{/if}}', 39 | 'The schemas provided are partial. The description provided may not convey full meaning.', 40 | 'This is preliminary feasibility check, you will have access to more data later - keep it loose', 41 | ], 42 | { 43 | bodySchema: JSON.stringify(params.requestSchema.body), 44 | querySchema: JSON.stringify(params.requestSchema.query), 45 | pathSchema: JSON.stringify(params.requestSchema.path), 46 | }, 47 | ); 48 | 49 | const { is_correct } = await this.llm.generateStructured({ 50 | messages: [ 51 | { 52 | content: message, 53 | role: 'user', 54 | }, 55 | ], 56 | model: ChatCompletionModels.trivial, 57 | generatorName: 'respond', 58 | generatorDescription: 'Did the user pick the right API?', 59 | generatorOutputSchema: Zod.object({ 60 | is_correct: Zod.boolean(), 61 | }), 62 | }); 63 | 64 | return is_correct; 65 | } 66 | 67 | async filterInputSchema( 68 | params: OperationMetdata & { 69 | requiredInputSchema?: JSONSchema7; 70 | inputSchema?: JSONSchema7; 71 | }, 72 | ) { 73 | let filteredSchema = params.requiredInputSchema ?? {}; 74 | const chunks = chunkSchema(params.inputSchema ?? {}); 75 | 76 | for (const { schema: chunkSchema, propertyNames } of chunks) { 77 | if (propertyNames.length === 0) continue; 78 | 79 | const messages: IChatCompletionMessage[] = [ 80 | { 81 | role: 'system', 82 | content: t([ 83 | "Your task is to identify and shortlist properties from the JSON schema that are relevant to the user's objective.", 84 | 'For instance, if the objective is "list emails for label `work`", and the JSON schema has properties like email, date, limit, label, etc.,', 85 | 'you should shortlist the "label" property because it is specifically referred to in the objective.', 86 | 'Avoid shortlisting properties that are not directly referred to in the objective or required by the schema, such as "email", "date", or "limit".', 87 | ]), 88 | }, 89 | { 90 | role: 'user', 91 | content: t( 92 | [ 93 | objectivePrefix(params), 94 | operationPrefix(params), 95 | 'The current request includes the following schema:', 96 | '```{{filteredSchema}}```', 97 | 'Your task is to shortlist the properties from the following JSONSchema that are relevant to achieving the objective:', 98 | '```{{chunkSchema}}```', 99 | 'A property is considered relevant if:', 100 | '- It is referred to in the objective', 101 | '- It is required by the JSON schema', 102 | '- Its values are mentioned in the objective or context', 103 | 'A property is considered irrelevant if:', 104 | '- It is not required by the objective or JSON schema', 105 | '- Its values are not mentioned in the objective or context', 106 | 'Remember to provide specific evidence for each shortlisted property. The evidence should refer to the objective or schema.', 107 | ], 108 | { 109 | filteredSchema: JSON.stringify(filteredSchema), 110 | chunkSchema: JSON.stringify(chunkSchema), 111 | }, 112 | ), 113 | }, 114 | ]; 115 | 116 | const { shortlist } = await this.llm.generateStructured({ 117 | messages: messages, 118 | model: ChatCompletionModels.critical, 119 | generatorName: 'shortlist_properties', 120 | generatorDescription: 121 | 'Identify and shortlist properties from the JSON schema that are relevant to the user objective', 122 | generatorOutputSchema: Zod.object({ 123 | shortlist: Zod.array( 124 | Zod.object({ 125 | propertyName: Zod.enum(propertyNames as [string]), 126 | evidence: Zod.string().describe( 127 | 'Provide a specific reason from the objective or schema that justifies the relevance of this property.', 128 | ), 129 | }), 130 | ).min(0), 131 | }), 132 | }); 133 | 134 | await this.logger.log('input shortlist', shortlist, chunkSchema); 135 | 136 | const subSchema = getSubSchema( 137 | chunkSchema, 138 | shortlist.map((s) => s.propertyName), 139 | ); 140 | filteredSchema = mergeSchema(filteredSchema, subSchema); 141 | } 142 | filteredSchema = deepenSchema(params.inputSchema ?? {}, filteredSchema); 143 | 144 | return filteredSchema; 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/agent/index.ts: -------------------------------------------------------------------------------- 1 | import { JSONSchema7 } from 'json-schema'; 2 | import { OpenAPI } from 'openapi-types'; 3 | import { JsonObject } from 'type-fest'; 4 | 5 | import CodeGenerator, { CodeGenLanguage } from '../agent/code-gen'; 6 | import ContextProcessor from '../agent/context-processor'; 7 | import ExternalResourceDirectory from '../agent/external-resource-directory'; 8 | import ExternalResourceEvaluator from '../agent/external-resource-evaluator'; 9 | import { EmbeddingFunctions } from '../embeddings'; 10 | import { LLM } from '../llm'; 11 | import { stringifyContext } from '../utils/context'; 12 | import Logger from '../utils/logger'; 13 | import { dereferencePath } from '../utils/openapi/dereference-path'; 14 | import { extractRequiredSchema } from '../utils/openapi/required-schema'; 15 | import { t } from '../utils/template'; 16 | 17 | export const objectivePrefix = ( 18 | params: Pick, 19 | withContext = true, 20 | ) => 21 | t( 22 | [ 23 | 'I am a backend engineer. I want to make an API call to achieve the following objective:', 24 | '```{{objective}}```.', 25 | '{{#if showContext}}Here is some historical context in JSON: ```{{context}}```{{/if}}', 26 | ], 27 | { 28 | ...params, 29 | context: stringifyContext(params.context), 30 | showContext: 31 | withContext && params.context && !!Object.keys(params.context).length, 32 | }, 33 | ); 34 | 35 | export const operationPrefix = ( 36 | params: Pick< 37 | OperationMetdata, 38 | 'provider' | 'path' | 'httpMethod' | 'description' 39 | >, 40 | skipFirstLine = false, 41 | ) => 42 | t( 43 | [ 44 | skipFirstLine ? '' : 'Chosen API:', 45 | 'Provider: {{provider}}', 46 | 'Path: {{path}}', 47 | 'Method: {{httpMethod}}', 48 | '{{#if description}}Description: {{description}}{{/if}}', 49 | ], 50 | params, 51 | ); 52 | 53 | export interface OperationMetdata { 54 | objective: string; 55 | context: JsonObject | null; 56 | provider: string; 57 | path: string; 58 | httpMethod: string; 59 | description: string; 60 | } 61 | 62 | export default class Parser { 63 | externalResourceDirectory: ExternalResourceDirectory; 64 | externalResourceEvaluator: ExternalResourceEvaluator; 65 | contextProcessor: ContextProcessor; 66 | codeGen: CodeGenerator; 67 | logger: Logger; 68 | 69 | constructor(args: { 70 | llm: LLM; 71 | logger: Logger; 72 | language?: CodeGenLanguage; 73 | embeddingFunctions: EmbeddingFunctions; 74 | }) { 75 | const { language = CodeGenLanguage.javascript, embeddingFunctions } = args; 76 | 77 | this.logger = args.logger; 78 | 79 | this.externalResourceDirectory = new ExternalResourceDirectory( 80 | args.logger, 81 | embeddingFunctions, 82 | args.llm, 83 | ); 84 | this.externalResourceEvaluator = new ExternalResourceEvaluator( 85 | args.logger, 86 | args.llm, 87 | ); 88 | this.contextProcessor = new ContextProcessor(args.logger, args.llm); 89 | this.codeGen = new CodeGenerator(args.logger, args.llm, language); 90 | } 91 | 92 | parse = async ( 93 | objective: string, 94 | context: Record, 95 | openAPIs: OpenAPI.Document[], 96 | ) => { 97 | const shortlist = await this.externalResourceDirectory.identify( 98 | objective, 99 | context, 100 | openAPIs, 101 | ); 102 | 103 | await this.logger.info('Here are the APIs we are shortlisting for you: \n'); 104 | await this.logger.log(shortlist); 105 | 106 | for (const api of shortlist) { 107 | const openapi = openAPIs.find((o) => o.info.title === api.provider)!; 108 | const operationObject = await dereferencePath( 109 | openapi, 110 | api.httpMethod, 111 | api.path, 112 | ); 113 | if (!operationObject) continue; 114 | 115 | const operationMetadata: OperationMetdata = { 116 | objective, 117 | context, 118 | provider: api.provider, 119 | path: api.path, 120 | method: api.httpMethod, 121 | description: api.description, 122 | }; 123 | 124 | const { 125 | bodySchema, 126 | querySchema, 127 | pathSchema, 128 | requestContentType, 129 | responseContentType, 130 | responseSchema, 131 | } = operationSchemas(operationObject); 132 | const requiredBodySchema = extractRequiredSchema(bodySchema); 133 | const requiredQuerySchema = extractRequiredSchema(querySchema); 134 | const requiredPathSchema = extractRequiredSchema(pathSchema); 135 | 136 | const [isFeasible, reason] = 137 | await this.externalResourceEvaluator.isFeasible({ 138 | ...operationMetadata, 139 | requiredInputSchema: { 140 | body: requiredBodySchema, 141 | query: requiredQuerySchema, 142 | path: requiredPathSchema, 143 | }, 144 | }); 145 | 146 | if (!isFeasible) { 147 | await this.logger.warn( 148 | `chosen API (${api.provider}: ${api.httpMethod} ${api.path}) is not feasible, moving on to the next API. reason: '${reason}'`, 149 | ); 150 | continue; 151 | } 152 | 153 | await this.logger.log( 154 | `Chosen API (${api.provider}: ${api.httpMethod} ${api.path}) is feasible, evaluating it further.`, 155 | ); 156 | 157 | await this.logger.log( 158 | 'Narrowing down the parameter schemas for the API...', 159 | ); 160 | 161 | const { 162 | body: filteredBodySchema, 163 | query: filteredQuerySchema, 164 | path: filteredPathSchema, 165 | } = await this.externalResourceEvaluator.filterInputSchemas({ 166 | ...operationMetadata, 167 | requiredInputSchema: { 168 | body: requiredBodySchema, 169 | query: requiredQuerySchema, 170 | path: requiredPathSchema, 171 | }, 172 | inputSchema: { 173 | body: bodySchema, 174 | query: querySchema, 175 | path: pathSchema, 176 | }, 177 | }); 178 | 179 | await this.logger.info({ 180 | filteredBodySchema, 181 | filteredQuerySchema, 182 | filteredPathSchema, 183 | }); 184 | 185 | await this.logger.log( 186 | 'Narrowing down the historical context that can be used in the API call...', 187 | ); 188 | 189 | const { 190 | filteredContextForBody, 191 | filteredContextForQuery, 192 | filteredContextForPath, 193 | } = await this.contextProcessor.filter({ 194 | ...operationMetadata, 195 | inputSchema: { 196 | body: filteredBodySchema, 197 | query: filteredQuerySchema, 198 | path: filteredPathSchema, 199 | }, 200 | }); 201 | 202 | await this.logger.info({ 203 | filteredContextForBody, 204 | filteredContextForQuery, 205 | filteredContextForPath, 206 | }); 207 | 208 | await this.logger.log('Generating the input parameters...'); 209 | 210 | const generatedCode = ( 211 | await Promise.allSettled([ 212 | this.codeGen.generateInputParamCode({ 213 | ...operationMetadata, 214 | inputSchema: filteredBodySchema, 215 | context: filteredContextForBody, 216 | name: 'body', 217 | }), 218 | this.codeGen.generateInputParamCode({ 219 | ...operationMetadata, 220 | inputSchema: filteredQuerySchema, 221 | context: filteredContextForQuery, 222 | name: 'query', 223 | }), 224 | this.codeGen.generateInputParamCode({ 225 | ...operationMetadata, 226 | inputSchema: filteredPathSchema, 227 | context: filteredContextForPath, 228 | name: 'path', 229 | }), 230 | ]) 231 | ).map((r) => { 232 | if (r.status === 'rejected') 233 | throw `couldnt generate code, reason: ${r.reason}`; 234 | 235 | return r.value; 236 | }); 237 | 238 | await this.logger.log( 239 | `Generating the ${this.codeGen.language} expression...`, 240 | ); 241 | 242 | return { 243 | provider: api.provider, 244 | method: api.httpMethod, 245 | path: api.path, 246 | bodyParams: generatedCode[0], 247 | queryParams: generatedCode[1], 248 | pathParams: generatedCode[2], 249 | servers: 'servers' in openapi ? openapi.servers : undefined, 250 | requestContentType, 251 | responseContentType, 252 | responseSchema, 253 | }; 254 | } 255 | 256 | throw 'couldnt find a feasible API'; 257 | }; 258 | } 259 | -------------------------------------------------------------------------------- /src/agent/external-resource-directory.ts: -------------------------------------------------------------------------------- 1 | import { reverse } from 'lodash'; 2 | import compact from 'lodash/compact'; 3 | import objecthash from 'object-hash'; 4 | import { OpenAPI, OpenAPIV2 } from 'openapi-types'; 5 | import { stripHtml } from 'string-strip-html'; 6 | import { JsonObject } from 'type-fest'; 7 | import Zod from 'zod'; 8 | 9 | import { objectivePrefix } from '../agent/index'; 10 | import { 11 | EmbeddingFunctions, 12 | InterveneParserItemMetadata, 13 | InterveneParserItemMetadataWhere, 14 | StorableInterveneParserItem, 15 | } from '../embeddings'; 16 | import { ChatCompletionModels, LLM } from '../llm'; 17 | import { ALPHABET } from '../utils/alphabet'; 18 | import { benchmark } from '../utils/benchmark'; 19 | import Logger from '../utils/logger'; 20 | import { dereferencePath } from '../utils/openapi/dereference-path'; 21 | import { 22 | extractOperationSchemas, 23 | getOauthSecuritySchemeName, 24 | getOperationScopes, 25 | } from '../utils/openapi/operation'; 26 | import { extractRequiredSchema } from '../utils/openapi/required-schema'; 27 | import { OpenAPITokenizer, TokenMap } from '../utils/openapi/tokenizer'; 28 | import { t } from '../utils/template'; 29 | 30 | export type OperationPath = string & { 31 | ____: never; 32 | split(separator: '|'): [string, string, OpenAPIV2.HttpMethods]; 33 | split(separator: string): string[]; 34 | }; 35 | 36 | export type APIMatch = { 37 | apiSpecId: string; 38 | scopes: string[]; 39 | httpMethod: OpenAPIV2.HttpMethods; 40 | path: string; 41 | description: string; 42 | }; 43 | 44 | export default class ExternalResourceDirectory { 45 | constructor( 46 | public logger: Logger, 47 | public embeddingFunctions: EmbeddingFunctions, 48 | public llm: LLM, 49 | ) {} 50 | 51 | /** 52 | * @param api the openapi document 53 | * @param id a unique identifier for the api document 54 | * @param provider the service provider (product/company) whom this api belongs to 55 | */ 56 | embed = async (api: OpenAPI.Document, id: string) => { 57 | await this.logger.log('Preparing embedding items for the openapi spec'); 58 | 59 | const tokenizer = new OpenAPITokenizer(id, api); 60 | const tokenMap = await tokenizer.tokenize(); 61 | 62 | await this.logger.log( 63 | `Embedding OpenAPI specs... ${tokenMap.size} items to embed`, 64 | ); 65 | 66 | // Process keys in batches 67 | await this.processKeysInBatches(tokenMap); 68 | 69 | await this.logger.log('All done with embedding, completed without errors.'); 70 | }; 71 | 72 | private async processKeysInBatches(tokenMap: TokenMap) { 73 | const itemIds = Array.from(tokenMap.keys()); 74 | const batchSize = 1000; 75 | const metadataMap = Object.fromEntries( 76 | itemIds.map((id) => { 77 | const metadata: InterveneParserItemMetadata = { 78 | paths: JSON.stringify(Array.from(tokenMap.get(id)!.paths)), 79 | apiSpecId: tokenMap.get(id)!.apiSpecId, 80 | tokens: tokenMap.get(id)!.tokens, 81 | }; 82 | 83 | tokenMap.get(id)!.scopes.forEach((scope) => { 84 | metadata[scope] = true; 85 | }); 86 | 87 | return [id, metadata]; 88 | }), 89 | ); 90 | const metadataHashMap = Object.fromEntries( 91 | itemIds.map((id) => [id, objecthash(metadataMap[id])]), 92 | ); 93 | 94 | // Process keys in batches 95 | for (let i = 0; i < itemIds.length; i += batchSize) { 96 | const idsBatch = itemIds.slice(i, i + batchSize).filter((key) => !!key); 97 | 98 | // Retrieve stored embeddings 99 | const storedEmbeddings = Object.fromEntries( 100 | (await this.embeddingFunctions.retrieveItems(idsBatch)).map((item) => [ 101 | item.id, 102 | item, 103 | ]), 104 | ); 105 | 106 | // Determine which keys need to be embedded 107 | const idsToEmbed = idsBatch.filter((id) => { 108 | if (!storedEmbeddings[id]) return true; 109 | 110 | if (metadataHashMap[id] !== storedEmbeddings[id].metadataHash) { 111 | return true; 112 | } 113 | 114 | return false; 115 | }); 116 | const tokensToEmbed = idsToEmbed 117 | .map((id) => tokenMap.get(id)!.tokens) 118 | .filter((t) => !!t); 119 | 120 | const embeddingsToStore: StorableInterveneParserItem[] = []; 121 | 122 | await this.logger.log('Embedding', tokensToEmbed.length, 'keys'); 123 | const embeddingsResponse = tokensToEmbed.length 124 | ? await this.embeddingFunctions.createEmbeddings(tokensToEmbed) 125 | : []; 126 | const embeddingsMap = Object.fromEntries(embeddingsResponse); 127 | 128 | for (const id of idsToEmbed) { 129 | const item = tokenMap.get(id)!; 130 | const embedding = embeddingsMap[item.tokens]; 131 | 132 | const metadata: InterveneParserItemMetadata = { 133 | paths: JSON.stringify(Array.from(tokenMap.get(id)!.paths)), 134 | apiSpecId: tokenMap.get(id)!.apiSpecId, 135 | tokens: item.tokens, 136 | }; 137 | 138 | tokenMap.get(id)!.scopes.forEach((scope) => { 139 | metadata[scope] = true; 140 | }); 141 | 142 | embeddingsToStore.push({ 143 | id: id, 144 | embeddings: embedding, 145 | metadataHash: objecthash(metadata), 146 | metadata, 147 | }); 148 | } 149 | 150 | await this.logger.log( 151 | 'Storing', 152 | embeddingsToStore.length, 153 | 'embeddings to store', 154 | ); 155 | 156 | await this.embeddingFunctions.upsertItems(embeddingsToStore); 157 | } 158 | } 159 | 160 | /** 161 | * 162 | * @param apiMap a map of api spec identifier to openapi document 163 | * @param scopes the scopes available to the user 164 | * @param objective the objective to accomplish 165 | * @param context the context of the objective 166 | */ 167 | identify = async ( 168 | apiMap: Record, 169 | scopes: string[], 170 | objective: string, 171 | context: JsonObject | null, 172 | ): Promise => { 173 | const [matches, shortObjective] = await this.query( 174 | apiMap, 175 | scopes, 176 | objective, 177 | ); 178 | await this.logger.log('Matches:', matches); 179 | const shortlist = await this.shortlist( 180 | matches, 181 | apiMap, 182 | context, 183 | shortObjective, 184 | ); 185 | await this.logger.log('Shortlist:', shortlist); 186 | 187 | return shortlist; 188 | }; 189 | 190 | private query = async ( 191 | apiMap: Record, 192 | scopes: string[], 193 | objective: string, 194 | ) => { 195 | const { documentation: shortObjective } = await benchmark( 196 | 'generating short objective', 197 | () => 198 | this.llm.generateStructured({ 199 | messages: [ 200 | { 201 | content: t( 202 | [ 203 | 'I am an engineer and my manager told to create an API that can be used to achieve this:', 204 | '```{{objective}}```', 205 | 'I have implemented the code and used relevant third party APIs.', 206 | "Your task is to generate the API documentation's description.", 207 | 'Rules:', 208 | '1. It is preferable not to mention third party vendors.', 209 | '2. It should not have fixed values. My boss may have given an example, replace it with the descriptions if required.', 210 | '3. It should atleast be 2 sentences long.', 211 | '4. It needs to be technical.', 212 | ], 213 | { objective }, 214 | ), 215 | role: 'user', 216 | }, 217 | ], 218 | model: ChatCompletionModels.trivial, 219 | generatorName: 'documentation_generator', 220 | generatorDescription: 'The generation for the API', 221 | generatorOutputSchema: Zod.object({ 222 | documentation: Zod.string(), 223 | }), 224 | }), 225 | this.logger, 226 | ); 227 | 228 | await this.logger.log('Decoded API documentation:', shortObjective); 229 | 230 | const embedding = await benchmark( 231 | 'create embedding for objective', 232 | () => this.embeddingFunctions.createEmbeddings([shortObjective]), 233 | this.logger, 234 | ); 235 | 236 | const where: InterveneParserItemMetadataWhere = { 237 | $or: scopes.map((scope) => ({ 238 | [scope]: true, 239 | })), 240 | }; 241 | 242 | const matches = await benchmark( 243 | `search objective in vector store`, 244 | () => 245 | this.embeddingFunctions.searchItems( 246 | shortObjective, 247 | embedding[0][1], 248 | 20, 249 | where, 250 | ), 251 | this.logger, 252 | ); 253 | 254 | const pathScores: Map = new Map(); 255 | const pathScopeMap = new Map(); 256 | 257 | // Iterate over each match 258 | for (const match of matches) { 259 | const matchPaths = JSON.parse(match.metadata.paths) as OperationPath[]; 260 | 261 | // Iterate over each path in the match 262 | for (const path of matchPaths) { 263 | const [apiSpecId, urlPath, httpMethod] = path.split('|'); 264 | const openapi = apiMap[apiSpecId]; 265 | const oauthSecuritySchemeName = getOauthSecuritySchemeName(openapi); 266 | const pathScopes = getOperationScopes( 267 | apiSpecId, 268 | openapi.paths![urlPath]![httpMethod]!, 269 | oauthSecuritySchemeName, 270 | ); 271 | const matchedScopes = scopes.filter((scope) => 272 | pathScopes.includes(scope), 273 | ); 274 | if (!matchedScopes.length) continue; 275 | 276 | // Get the current score of the path or default to 0 if it doesn't exist 277 | const score = pathScores.get(path) ?? 0; 278 | // Update the score of the path 279 | pathScores.set(path, score + (1 - (match.distance ?? 0))); 280 | pathScopeMap.set(path, matchedScopes); 281 | } 282 | } 283 | 284 | // Sort the paths based on their scores 285 | const sortedPaths = Array.from(pathScores.entries()).sort( 286 | (a, b) => b[1] - a[1], 287 | ); 288 | 289 | // Get the top 15 paths 290 | const matchingPaths = sortedPaths 291 | .map((entry) => entry[0]) 292 | .splice(0, 7) as OperationPath[]; 293 | 294 | return [ 295 | compact( 296 | matchingPaths.map((path) => { 297 | const [apiSpecId, pathName, httpMethod] = path.split('|'); 298 | const operationObject = 299 | apiMap[apiSpecId].paths?.[pathName]?.[httpMethod]; 300 | 301 | if (!operationObject) return; 302 | 303 | const description = 304 | operationObject.description ?? 305 | operationObject.summary ?? 306 | operationObject.operationId ?? 307 | path; 308 | 309 | return { 310 | apiSpecId, 311 | scopes: pathScopeMap.get(path)!, 312 | httpMethod: httpMethod, 313 | path: pathName, 314 | description, 315 | } satisfies APIMatch; 316 | }), 317 | ), 318 | shortObjective, 319 | ] as const; 320 | }; 321 | 322 | private shortlist = async ( 323 | matches: APIMatch[], 324 | apiMap: Record, 325 | context: JsonObject | null, 326 | objective: string, 327 | ) => { 328 | const matchDetails: { 329 | provider: string; 330 | httpMethod: OpenAPIV2.HttpMethods; 331 | path: string; 332 | description: string; 333 | }[] = []; 334 | 335 | for (const match of matches) { 336 | const openapi = apiMap[match.apiSpecId]; 337 | const operationObject = openapi?.paths?.[match.path]?.[match.httpMethod]; 338 | if (!operationObject) continue; 339 | 340 | matchDetails.push({ 341 | provider: openapi.info.title, 342 | httpMethod: match.httpMethod, 343 | path: match.path, 344 | description: stripHtml( 345 | operationObject.description ?? 346 | operationObject.summary ?? 347 | operationObject.operationId ?? 348 | match.path, 349 | ).result, 350 | }); 351 | } 352 | 353 | const matchesStr = reverse(matchDetails).map( 354 | ({ provider, httpMethod, path, description }) => { 355 | return `${provider}: ${httpMethod.toUpperCase()} ${path}\n'${description}'`; 356 | }, 357 | ); 358 | 359 | const message = t( 360 | [ 361 | objectivePrefix({ objective, context }), 362 | 'Your task is to shortlist APIs that can be used to accomplish the objective', 363 | 'Here is a list of possible choices for the API call:\n', 364 | '{{#each matchesStr}}', 365 | '{{getAlphabet @index}}. {{this}}\n', 366 | '{{/each}}', 367 | 'Your task is to shortlist at most 3 APIs in descending order of fittingness.', 368 | ], 369 | { 370 | matchesStr, 371 | }, 372 | ); 373 | 374 | await this.logger.log('Shortlisting indexes:', message); 375 | 376 | let { indexes: shortlistedIndexes } = await benchmark( 377 | 'shortlist APIs that might work out for the objective', 378 | () => 379 | this.llm.generateStructured({ 380 | messages: [ 381 | { 382 | content: message, 383 | role: 'user', 384 | }, 385 | ], 386 | model: ChatCompletionModels.trivial, 387 | generatorName: 'api_shortlist', 388 | generatorDescription: 389 | 'Shortlist APIs that might work out for the objective.', 390 | generatorOutputSchema: Zod.object({ 391 | indexes: Zod.array( 392 | Zod.object({ 393 | index: Zod.enum( 394 | ALPHABET.slice(0, matchesStr.length) as [string], 395 | ).describe('The index of API in the given list'), 396 | score: Zod.number() 397 | .describe('How good fit is this API?') 398 | .min(0) 399 | .max(10), 400 | }), 401 | ), 402 | }), 403 | }), 404 | this.logger, 405 | ); 406 | await this.logger.log( 407 | 'Shortlisted indexes:', 408 | JSON.stringify(shortlistedIndexes), 409 | ); 410 | shortlistedIndexes = shortlistedIndexes.sort((a, b) => b.score - a.score); 411 | 412 | const filteredMatches: APIMatch[] = []; 413 | 414 | for (const matchI of shortlistedIndexes) { 415 | const match = matches[ALPHABET.indexOf(matchI.index)]; 416 | if (match) filteredMatches.push(match); 417 | } 418 | 419 | return filteredMatches; 420 | }; 421 | 422 | async extractOperationComponents( 423 | api: OpenAPI.Document, 424 | pathName: string, 425 | httpMethod: OpenAPIV2.HttpMethods, 426 | ) { 427 | const operationObject = await dereferencePath(api, httpMethod, pathName); 428 | if (!operationObject) { 429 | throw `Could not find operation object for ${httpMethod.toUpperCase()} ${pathName}`; 430 | } 431 | 432 | const { 433 | bodySchema, 434 | querySchema, 435 | pathSchema, 436 | requestContentType, 437 | responseContentType, 438 | responseSchema, 439 | } = extractOperationSchemas(operationObject); 440 | 441 | const requiredBodySchema = extractRequiredSchema(bodySchema); 442 | const requiredQuerySchema = extractRequiredSchema(querySchema); 443 | const requiredPathSchema = extractRequiredSchema(pathSchema); 444 | 445 | return { 446 | operationObject, 447 | requestSchema: { 448 | body: bodySchema, 449 | query: querySchema, 450 | path: pathSchema, 451 | contentType: requestContentType, 452 | required: { 453 | body: requiredBodySchema, 454 | query: requiredQuerySchema, 455 | path: requiredPathSchema, 456 | }, 457 | }, 458 | response: { 459 | contentType: responseContentType, 460 | schema: responseSchema, 461 | }, 462 | }; 463 | } 464 | } 465 | --------------------------------------------------------------------------------