├── template └── typescript │ ├── .eslintignore │ ├── .env.example │ ├── src │ ├── types.ts │ ├── flow.ts │ ├── utils │ │ └── callLlm.ts │ ├── index.ts │ └── nodes.ts │ ├── vitest.config.mts │ ├── .prettierrc │ ├── tsup.config.ts │ ├── tsconfig.json │ ├── eslint.config.mjs │ ├── .gitignore │ ├── package.json │ ├── README.md │ ├── docs │ └── design.md │ └── .cursorrules ├── .simple-git-hooks.json ├── .prettierrc ├── .prettierignore ├── lib ├── index.js ├── utils │ ├── format.js │ └── logger.js ├── package-manager.js ├── template-manager.js └── create-project.js ├── bin └── cli.js ├── .gitignore ├── package.json └── README.md /template/typescript/.eslintignore: -------------------------------------------------------------------------------- 1 | jest.config.js 2 | -------------------------------------------------------------------------------- /template/typescript/.env.example: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY=your_openai_api_key -------------------------------------------------------------------------------- /.simple-git-hooks.json: -------------------------------------------------------------------------------- 1 | { 2 | "pre-commit": "npx pretty-quick --staged" 3 | } 4 | -------------------------------------------------------------------------------- /template/typescript/src/types.ts: -------------------------------------------------------------------------------- 1 | export interface QASharedStore { 2 | question?: string 3 | answer?: string 4 | } 5 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": true, 3 | "tabWidth": 4, 4 | "singleQuote": true, 5 | "trailingComma": "all", 6 | "printWidth": 120 7 | } 8 | -------------------------------------------------------------------------------- /template/typescript/vitest.config.mts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitest/config' 2 | 3 | export default defineConfig({ 4 | test: { 5 | }, 6 | }) 7 | -------------------------------------------------------------------------------- /template/typescript/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": false, 3 | "tabWidth": 2, 4 | "singleQuote": true, 5 | "trailingComma": "all", 6 | "printWidth": 120 7 | } 8 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | 4 | # Build output 5 | dist/ 6 | build/ 7 | 8 | # Coverage directory 9 | coverage/ 10 | 11 | # Template 12 | template/ -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | export { default as createProject } from './create-project.js'; 2 | export { default as logger } from './utils/logger.js'; 3 | export { default as templateManager } from './template-manager.js'; 4 | -------------------------------------------------------------------------------- /template/typescript/tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'tsup' 2 | 3 | export default defineConfig({ 4 | entry: ['src/index.ts'], 5 | format: ['cjs', 'esm'], 6 | dts: true, 7 | splitting: false, 8 | sourcemap: true, 9 | clean: true, 10 | }) 11 | -------------------------------------------------------------------------------- /template/typescript/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "emitDeclarationOnly": true, 4 | "declaration": true, 5 | "target": "ESNext", 6 | "newLine": "LF", 7 | "module": "NodeNext", 8 | "moduleResolution": "NodeNext", 9 | "strict": true, 10 | "jsx": "preserve" 11 | }, 12 | "exclude": ["node_modules"] 13 | } -------------------------------------------------------------------------------- /bin/cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import { program } from 'commander'; 3 | import { createProject } from '../lib/index.js'; 4 | 5 | program 6 | .name('create-pocketflow') 7 | .description('Create a new PocketFlow project') 8 | .argument('[project-directory]', 'Project directory name') 9 | .action(async (projectDir) => { 10 | await createProject(projectDir); 11 | }); 12 | 13 | program.parse(process.argv); 14 | -------------------------------------------------------------------------------- /lib/utils/format.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @param {string[]} templates 4 | * @returns {Array<{name: string, value: string}>} 5 | */ 6 | export const formatAvailableTemplates = (templates) => { 7 | return templates.map((value) => { 8 | const name = capitalize(value); 9 | return { 10 | name, 11 | value, 12 | }; 13 | }); 14 | }; 15 | 16 | const capitalize = (str) => { 17 | return str[0].toUpperCase() + str.slice(1); 18 | }; 19 | -------------------------------------------------------------------------------- /template/typescript/eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'eslint/config' 2 | import globals from 'globals' 3 | import js from '@eslint/js' 4 | import tseslint from 'typescript-eslint' 5 | 6 | export default defineConfig([ 7 | { files: ['**/*.{js,mjs,cjs,ts}'] }, 8 | { files: ['**/*.{js,mjs,cjs,ts}'], languageOptions: { globals: globals.browser } }, 9 | { files: ['**/*.{js,mjs,cjs,ts}'], plugins: { js }, extends: ['js/recommended'] }, 10 | tseslint.configs.recommended, 11 | ]) 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | /node_modules 3 | 4 | # Build output 5 | dist/ 6 | build/ 7 | 8 | # Coverage directory 9 | coverage/ 10 | 11 | # Logs 12 | logs 13 | *.log 14 | npm-debug.log* 15 | yarn-debug.log* 16 | yarn-error.log* 17 | 18 | # Environment variables 19 | .env 20 | .env.local 21 | .env.development.local 22 | .env.test.local 23 | .env.production.local 24 | 25 | # IDE files 26 | .idea/ 27 | .vscode/ 28 | *.swp 29 | *.swo 30 | 31 | # OS specific 32 | .DS_Store 33 | Thumbs.db -------------------------------------------------------------------------------- /template/typescript/src/flow.ts: -------------------------------------------------------------------------------- 1 | import { Flow } from 'pocketflow' 2 | import { GetQuestionNode, AnswerNode } from './nodes' 3 | import type { QASharedStore } from './types' 4 | 5 | export function createQaFlow(): Flow { 6 | // Create nodes 7 | const getQuestionNode = new GetQuestionNode() 8 | const answerNode = new AnswerNode() 9 | 10 | // Connect nodes in sequence 11 | getQuestionNode.next(answerNode) 12 | 13 | // Create flow starting with input node 14 | return new Flow(getQuestionNode) 15 | } 16 | -------------------------------------------------------------------------------- /template/typescript/.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | node_modules/ 3 | 4 | # Build output 5 | dist/ 6 | build/ 7 | lib/ 8 | 9 | # Coverage directory 10 | coverage/ 11 | 12 | # Logs 13 | logs 14 | *.log 15 | npm-debug.log* 16 | yarn-debug.log* 17 | yarn-error.log* 18 | 19 | # Environment variables 20 | .env 21 | .env.local 22 | .env.development.local 23 | .env.test.local 24 | .env.production.local 25 | 26 | # IDE files 27 | .idea/ 28 | .vscode/ 29 | *.swp 30 | *.swo 31 | 32 | # OS specific 33 | .DS_Store 34 | Thumbs.db -------------------------------------------------------------------------------- /template/typescript/src/utils/callLlm.ts: -------------------------------------------------------------------------------- 1 | import { OpenAI } from 'openai' 2 | import 'dotenv/config' 3 | 4 | export async function callLlm(prompt: string): Promise { 5 | const apiKey = process.env.OPENAI_API_KEY 6 | 7 | if (!apiKey) { 8 | throw new Error('OPENAI_API_KEY environment variable is not set') 9 | } 10 | 11 | const client = new OpenAI({ apiKey }) 12 | const r = await client.chat.completions.create({ 13 | model: 'gpt-4o', 14 | messages: [{ role: 'user', content: prompt }], 15 | }) 16 | return r.choices[0].message.content || '' 17 | } 18 | -------------------------------------------------------------------------------- /template/typescript/src/index.ts: -------------------------------------------------------------------------------- 1 | import 'dotenv/config' 2 | import { createQaFlow } from './flow' 3 | import type { QASharedStore } from './types' 4 | 5 | // Example main function 6 | async function main(): Promise { 7 | const shared: QASharedStore = { 8 | question: undefined, // Will be populated by GetQuestionNode from user input 9 | answer: undefined, // Will be populated by AnswerNode 10 | } 11 | 12 | // Create the flow and run it 13 | const qaFlow = createQaFlow() 14 | await qaFlow.run(shared) 15 | console.log(`Question: ${shared.question}`) 16 | console.log(`Answer: ${shared.answer}`) 17 | } 18 | 19 | // Run the main function 20 | main().catch(console.error) 21 | -------------------------------------------------------------------------------- /lib/utils/logger.js: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | 3 | const logger = { 4 | log: (message) => { 5 | console.log(message); 6 | }, 7 | 8 | info: (message) => { 9 | console.log(chalk.blue('Info: '), message); 10 | }, 11 | 12 | success: (message) => { 13 | console.log(chalk.green('Success!'), message); 14 | }, 15 | 16 | error: (message) => { 17 | console.log(chalk.red('Error:'), message); 18 | }, 19 | 20 | warn: (message) => { 21 | console.log(chalk.yellow('Warning:'), message); 22 | }, 23 | 24 | /** 25 | * log a command and its description 26 | * @param {string} command 27 | * @param {string} description - the description of the command, optional 28 | */ 29 | command: (command, description) => { 30 | console.log(chalk.cyan(` ${command}`)); 31 | description && console.log(` ${description}`); 32 | }, 33 | }; 34 | 35 | export default logger; 36 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "create-pocketflow", 3 | "version": "0.1.3", 4 | "description": "CLI tool to create PocketFlow projects", 5 | "main": "index.js", 6 | "type": "module", 7 | "bin": { 8 | "create-pocketflow": "./bin/cli.js" 9 | }, 10 | "files": [ 11 | "bin", 12 | "lib", 13 | "template" 14 | ], 15 | "scripts": { 16 | "prettier:all": "prettier --write \"**/*.js\"" 17 | }, 18 | "keywords": [ 19 | "pocketflow", 20 | "create", 21 | "template", 22 | "cli" 23 | ], 24 | "author": "", 25 | "license": "MIT", 26 | "dependencies": { 27 | "chalk": "^4.1.2", 28 | "commander": "^9.4.1", 29 | "fs-extra": "^11.1.1", 30 | "inquirer": "^8.2.5" 31 | }, 32 | "engines": { 33 | "node": ">=18.0.0" 34 | }, 35 | "devDependencies": { 36 | "prettier": "^3.5.3", 37 | "pretty-quick": "^4.1.1", 38 | "simple-git-hooks": "^2.13.0" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Create PocketFlow 2 | 3 | A CLI tool to easily create PocketFlow projects using the TypeScript template. 4 | 5 | ## Usage 6 | 7 | ```bash 8 | npx create-pocketflow 9 | # or 10 | npx create-pocketflow my-app 11 | ``` 12 | 13 | ## Features 14 | 15 | - Creates a new PocketFlow project with a TypeScript template 16 | - Interactive command-line interface for customizing project setup 17 | - Sets up all the necessary dependencies and structure 18 | 19 | ## Development 20 | 21 | ```bash 22 | # Clone the repository 23 | git clone https://github.com/yourusername/create-pocketflow.git 24 | cd create-pocketflow 25 | 26 | # Install dependencies 27 | npm install 28 | 29 | # Link the package globally for local testing 30 | npm link 31 | 32 | # Now you can use it like this 33 | create-pocketflow my-test-app 34 | ``` 35 | 36 | ## Publishing 37 | 38 | ```bash 39 | # Login to npm 40 | npm login 41 | 42 | # Publish the package 43 | npm publish 44 | ``` 45 | 46 | ## Future Features 47 | 48 | - [ ] Support for JavaScript template 49 | - [x] Support different package managers (yarn, pnpm, bun, deno etc.) 50 | - [ ] Support for different project types (library, full stack, api etc.) 51 | 52 | ## License 53 | 54 | MIT 55 | -------------------------------------------------------------------------------- /template/typescript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pocketflow-template-typescript", 3 | "version": "0.1.0", 4 | "description": "TypeScript template for PocketFlow", 5 | "main": "dist/index.js", 6 | "types": "dist/index.d.ts", 7 | "scripts": { 8 | "build": "tsup", 9 | "dev": "ts-node src/index.ts", 10 | "start": "npm run build && node dist/index.js", 11 | "test": "vitest", 12 | "lint": "eslint src --ext .ts,.js,.jsx,.tsx" 13 | }, 14 | "keywords": [ 15 | "pocketflow", 16 | "template", 17 | "typescript" 18 | ], 19 | "author": "", 20 | "license": "MIT", 21 | "devDependencies": { 22 | "@eslint/js": "^9.23.0", 23 | "@types/node": "^18.16.0", 24 | "@types/prompt-sync": "^4.2.3", 25 | "@typescript-eslint/eslint-plugin": "^8.28.0", 26 | "@typescript-eslint/parser": "^8.28.0", 27 | "eslint": "^9.23.0", 28 | "globals": "^16.0.0", 29 | "prettier": "^2.8.8", 30 | "prompt-sync": "^4.2.0", 31 | "ts-node": "^10.9.2", 32 | "tsup": "^8.4.0", 33 | "typescript": "^5.8.2", 34 | "typescript-eslint": "^8.28.0", 35 | "vitest": "^3.0.9" 36 | }, 37 | "dependencies": { 38 | "dotenv": "^16.4.7", 39 | "openai": "^4.90.0", 40 | "pocketflow": "^1.0.4" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /template/typescript/src/nodes.ts: -------------------------------------------------------------------------------- 1 | import { Node } from 'pocketflow' 2 | import { callLlm } from './utils/callLlm' 3 | import type { QASharedStore } from './types' 4 | import PromptSync from 'prompt-sync' 5 | 6 | const prompt = PromptSync() 7 | 8 | export class GetQuestionNode extends Node { 9 | async exec(): Promise { 10 | // Get question directly from user input 11 | const userQuestion = prompt('Enter your question: ') || '' 12 | return userQuestion 13 | } 14 | 15 | async post( 16 | shared: QASharedStore, 17 | _: unknown, 18 | execRes: string, 19 | ): Promise { 20 | // Store the user's question 21 | shared.question = execRes 22 | return 'default' // Go to the next node 23 | } 24 | } 25 | 26 | export class AnswerNode extends Node { 27 | async prep(shared: QASharedStore): Promise { 28 | // Read question from shared 29 | return shared.question || '' 30 | } 31 | 32 | async exec(question: string): Promise { 33 | // Call LLM to get the answer 34 | return await callLlm(question) 35 | } 36 | 37 | async post( 38 | shared: QASharedStore, 39 | _: unknown, 40 | execRes: string, 41 | ): Promise { 42 | // Store the answer in shared 43 | shared.answer = execRes 44 | return undefined 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /lib/package-manager.js: -------------------------------------------------------------------------------- 1 | import { execSync } from 'child_process'; 2 | 3 | const packageManager = { 4 | /** 5 | * List of package managers 6 | */ 7 | managers: [ 8 | { name: 'npm', installCmd: 'npm install', runCmd: 'npm run' }, 9 | { name: 'yarn', installCmd: 'yarn install', runCmd: 'yarn run' }, 10 | { name: 'pnpm', installCmd: 'pnpm install', runCmd: 'pnpm run' }, 11 | { name: 'bun', installCmd: 'bun install', runCmd: 'bun run' }, 12 | { name: 'deno', installCmd: 'deno install', runCmd: 'deno run' }, 13 | ], 14 | 15 | /** 16 | * Detect package manager installed in the system 17 | * @param {string} packageName 18 | * @returns {boolean} 19 | */ 20 | detect: (packageName) => { 21 | try { 22 | execSync(`${packageName} --version`, { stdio: 'ignore' }); 23 | return true; 24 | } catch (error) { 25 | return false; 26 | } 27 | }, 28 | 29 | getManagerNames: () => { 30 | return packageManager.managers.map((pm) => pm.name); 31 | }, 32 | 33 | getManagerByName: (name) => { 34 | const manager = packageManager.managers.find((pm) => pm.name === name) || null; 35 | const isIntsalled = packageManager.detect(name); 36 | if (!isIntsalled) { 37 | throw new Error(`${name} is not installed. Please install it first.`); 38 | } 39 | return manager; 40 | }, 41 | }; 42 | 43 | export default packageManager; 44 | -------------------------------------------------------------------------------- /template/typescript/README.md: -------------------------------------------------------------------------------- 1 | # PocketFlow-Template-Typescript 2 | 3 | A TypeScript template for creating PocketFlow applications. 4 | 5 | ## Features 6 | 7 | - TypeScript configuration 8 | - ESLint and Prettier setup 9 | - Vitest testing framework 10 | - Basic project structure 11 | - Example utility functions and types 12 | 13 | ## Getting Started 14 | 15 | ### Development 16 | 17 | ```bash 18 | # Install dependencies 19 | npm install 20 | 21 | # Build the project 22 | npm run build 23 | 24 | # Run development mode with watch 25 | npm run dev 26 | 27 | # Run tests 28 | npm test 29 | 30 | # Lint code 31 | npm run lint 32 | ``` 33 | 34 | ## Project Structure 35 | 36 | ``` 37 | . 38 | ├── src/ 39 | │ ├── index.ts # Main entry point 40 | │ ├── types.ts # Type definitions 41 | │ ├── nodes.ts # Node definitions 42 | │ ├── flow.ts # Flow related functionality 43 | │ └── utils/ # Utility functions 44 | │ └── callLlm.ts # LLM API integration 45 | ├── docs/ # Documentation 46 | │ └── design.md # Design documentation 47 | ├── dist/ # Compiled output 48 | ├── tsup.config.ts # Build configuration 49 | ├── eslint.config.mjs # ESLint configuration 50 | ├── vitest.config.ts # Vitest configuration 51 | ├── package.json # Project dependencies and scripts 52 | ├── tsconfig.json # TypeScript configuration 53 | ├── .env.example # Example environment variables 54 | └── README.md # Project documentation 55 | ``` 56 | 57 | ## Customizing the Template 58 | 59 | You can customize this template to fit your specific needs by: 60 | 61 | 1. Modifying the TypeScript configuration in `tsconfig.json` 62 | 2. Updating ESLint rules in `eslint.config.mjs` 63 | 3. Configuring the build process in `tsup.config.ts` 64 | 4. Adding more dependencies to `package.json` 65 | 5. Setting up environment variables using `.env` (see `.env.example`) 66 | 67 | ## License 68 | 69 | MIT 70 | -------------------------------------------------------------------------------- /template/typescript/docs/design.md: -------------------------------------------------------------------------------- 1 | # Design Doc: Your Project Name 2 | 3 | > Please DON'T remove notes for AI 4 | 5 | ## Requirements 6 | 7 | > Notes for AI: Keep it simple and clear. 8 | > If the requirements are abstract, write concrete user stories 9 | 10 | ## Flow Design 11 | 12 | > Notes for AI: 13 | > 14 | > 1. Consider the design patterns of agent, map-reduce, rag, and workflow. Apply them if they fit. 15 | > 2. Present a concise, high-level description of the workflow. 16 | 17 | ### Applicable Design Pattern: 18 | 19 | 1. Map the file summary into chunks, then reduce these chunks into a final summary. 20 | 2. Agentic file finder 21 | - _Context_: The entire summary of the file 22 | - _Action_: Find the file 23 | 24 | ### Flow high-level Design: 25 | 26 | 1. **First Node**: This node is for ... 27 | 2. **Second Node**: This node is for ... 28 | 3. **Third Node**: This node is for ... 29 | 30 | ```mermaid 31 | flowchart TD 32 | firstNode[First Node] --> secondNode[Second Node] 33 | secondNode --> thirdNode[Third Node] 34 | ``` 35 | 36 | ## Utility Functions 37 | 38 | > Notes for AI: 39 | > 40 | > 1. Understand the utility function definition thoroughly by reviewing the doc. 41 | > 2. Include only the necessary utility functions, based on nodes in the flow. 42 | 43 | 1. **Call LLM** (`src/utils/callLlm.ts`) 44 | 45 | - _Input_: prompt (str) 46 | - _Output_: response (str) 47 | - Generally used by most nodes for LLM tasks 48 | 49 | 2. **Embedding** (`src/utils/getEmbedding.ts`) 50 | - _Input_: str 51 | - _Output_: a vector of 3072 floats 52 | - Used by the second node to embed text 53 | 54 | ## Node Design 55 | 56 | ### Shared Memory 57 | 58 | > Notes for AI: Try to minimize data redundancy 59 | 60 | The shared memory structure is organized as follows: 61 | 62 | ```typescript 63 | interface SharedMemory { 64 | key: string; 65 | } 66 | ``` 67 | 68 | ### Node Steps 69 | 70 | > Notes for AI: Carefully decide whether to use Batch/Node/Flow. 71 | 72 | 1. First Node 73 | 74 | - _Purpose_: Provide a short explanation of the node’s function 75 | - _Type_: Decide between Regular, Batch, or Async 76 | - _Steps_: 77 | - _prep_: Read "key" from the shared store 78 | - _exec_: Call the utility function 79 | - _post_: Write "key" to the shared store 80 | 81 | 2. Second Node 82 | ... 83 | -------------------------------------------------------------------------------- /lib/template-manager.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import fs from 'fs-extra'; 3 | 4 | const templateManager = { 5 | /** 6 | * Get available templates names 7 | * @param {string} templatesDir 8 | * @returns {string[]} available templates names 9 | */ 10 | getAvailableTemplates: (templatesDir) => { 11 | try { 12 | return fs.readdirSync(templatesDir).filter((file) => { 13 | return fs.statSync(path.join(templatesDir, file)).isDirectory(); 14 | }); 15 | } catch (error) { 16 | return []; 17 | } 18 | }, 19 | 20 | /** 21 | * Copy template to target directory 22 | * @param {string} templatePath 23 | * @param {string} targetDir 24 | * @returns {Promise} 25 | */ 26 | copyTemplate: async (templatePath, targetDir) => { 27 | return await fs.copy(templatePath, targetDir); 28 | }, 29 | 30 | /** 31 | * Update package.json in target directory 32 | * @param {string} targetDir 33 | * @param {Object} updates fields to update in package.json 34 | * @returns {Promise} 35 | */ 36 | updatePackageJson: async (targetDir, updates) => { 37 | const pkgJsonPath = path.join(targetDir, 'package.json'); 38 | 39 | if (await fs.pathExists(pkgJsonPath)) { 40 | const pkgJson = await fs.readJson(pkgJsonPath); 41 | const updatedPkgJson = { ...pkgJson, ...updates }; 42 | await fs.writeJson(pkgJsonPath, updatedPkgJson, { spaces: 2 }); 43 | } 44 | }, 45 | 46 | /** 47 | * Update package.json scripts in target directory 48 | * @param {string} targetDir 49 | * @param {Object} packageManager 50 | * @param {string} packageManager.runCmd 51 | * @param {string} packageManager.installCmd 52 | * @param {string} packageManager.name 53 | * @returns {Promise} 54 | */ 55 | updatePackageScripts: async (targetDir, packageManager) => { 56 | const pkgJsonPath = path.join(targetDir, 'package.json'); 57 | 58 | if (await fs.pathExists(pkgJsonPath)) { 59 | const pkgJson = await fs.readJson(pkgJsonPath); 60 | 61 | // need to update scripts if package manager is not npm 62 | if (packageManager.name !== 'npm' && pkgJson.scripts) { 63 | const scripts = { ...pkgJson.scripts }; 64 | 65 | Object.keys(scripts).forEach((key) => { 66 | if (scripts[key].includes('npm run')) { 67 | scripts[key] = scripts[key].replace('npm run', packageManager.runCmd); 68 | } 69 | if (scripts[key].includes('npm install')) { 70 | scripts[key] = scripts[key].replace('npm install', packageManager.installCmd); 71 | } 72 | }); 73 | 74 | pkgJson.scripts = scripts; 75 | await fs.writeJson(pkgJsonPath, pkgJson, { spaces: 2 }); 76 | } 77 | } 78 | }, 79 | }; 80 | 81 | export default templateManager; 82 | -------------------------------------------------------------------------------- /lib/create-project.js: -------------------------------------------------------------------------------- 1 | import { fileURLToPath } from 'url'; 2 | import path from 'path'; 3 | import fs from 'fs-extra'; 4 | import inquirer from 'inquirer'; 5 | import chalk from 'chalk'; 6 | import logger from './utils/logger.js'; 7 | import templateManager from './template-manager.js'; 8 | import { formatAvailableTemplates } from './utils/format.js'; 9 | import pakageManager from './package-manager.js'; 10 | 11 | const __filename = fileURLToPath(import.meta.url); 12 | const __dirname = path.dirname(__filename); 13 | 14 | /** 15 | * Create a new project 16 | * @param {string} projectDir Project Dir and Project Name 17 | * @returns {Promise} 18 | */ 19 | async function createProject(projectDir) { 20 | try { 21 | // Ask for project name if not provided 22 | if (!projectDir) { 23 | const answers = await inquirer.prompt([ 24 | { 25 | type: 'input', 26 | name: 'projectDir', 27 | message: 'What is your project named?', 28 | default: 'my-pocketflow-app', 29 | }, 30 | ]); 31 | projectDir = answers.projectDir; 32 | } 33 | 34 | const targetDir = path.resolve(process.cwd(), projectDir); 35 | 36 | // Check if directory exists 37 | if (fs.existsSync(targetDir)) { 38 | const { overwrite } = await inquirer.prompt([ 39 | { 40 | type: 'confirm', 41 | name: 'overwrite', 42 | message: `Directory ${projectDir} already exists. Do you want to overwrite it?`, 43 | default: false, 44 | }, 45 | ]); 46 | 47 | if (!overwrite) { 48 | logger.error('Operation cancelled'); 49 | process.exit(1); 50 | } 51 | 52 | await fs.remove(targetDir); 53 | } 54 | 55 | // Create project directory 56 | await fs.ensureDir(targetDir); 57 | 58 | // Select package manager 59 | const { packageName } = await inquirer.prompt([ 60 | { 61 | type: 'list', 62 | name: 'packageName', 63 | message: 'Select a package manager:', 64 | choices: pakageManager.getManagerNames(), 65 | default: 'npm', 66 | }, 67 | ]); 68 | 69 | const selectedPackageManager = pakageManager.getManagerByName(packageName); 70 | 71 | // Select template 72 | const templatesDir = path.resolve(__dirname, '../template'); 73 | const { template } = await inquirer.prompt([ 74 | { 75 | type: 'list', 76 | name: 'template', 77 | message: 'Select a template:', 78 | choices: formatAvailableTemplates(templateManager.getAvailableTemplates(templatesDir)), 79 | default: 'typescript', 80 | }, 81 | ]); 82 | 83 | // Copy template 84 | const templateDir = path.join(templatesDir, template); 85 | logger.info(`Creating a new PocketFlow project in ${chalk.green(targetDir)}...\n`); 86 | 87 | await templateManager.copyTemplate(templateDir, targetDir); 88 | 89 | // Update package.json in the new project 90 | await templateManager.updatePackageJson(targetDir, { 91 | name: projectDir.toLowerCase().replace(/\s+/g, '-'), 92 | }); 93 | 94 | // Update package scripts in the new project 95 | await templateManager.updatePackageScripts(targetDir, selectedPackageManager); 96 | 97 | logger.success(`Created ${chalk.cyan(projectDir)} at ${chalk.cyan(targetDir)}\n`); 98 | 99 | logger.log('Inside that directory, you can run several commands:\n'); 100 | logger.command(selectedPackageManager.installCmd, 'Install the dependencies\n'); 101 | logger.command(`${selectedPackageManager.runCmd} dev`, 'Start the development server\n'); 102 | 103 | logger.log('We suggest that you begin by typing:\n'); 104 | logger.command(`cd ${chalk.reset(projectDir)}`); 105 | logger.command(selectedPackageManager.installCmd); 106 | logger.command(`${selectedPackageManager.runCmd} dev`); 107 | logger.log(''); 108 | 109 | logger.log('Happy coding!'); 110 | } catch (error) { 111 | logger.error(error.message); 112 | process.exit(1); 113 | } 114 | } 115 | 116 | export default createProject; 117 | -------------------------------------------------------------------------------- /template/typescript/.cursorrules: -------------------------------------------------------------------------------- 1 | ================================================ 2 | File: docs/guide.md 3 | ================================================ 4 | 5 | --- 6 | 7 | layout: default 8 | title: "Agentic Coding" 9 | 10 | --- 11 | 12 | # Agentic Coding: Humans Design, Agents code! 13 | 14 | > If you are an AI agents involved in building LLM Systems, read this guide **VERY, VERY** carefully! This is the most important chapter in the entire document. Throughout development, you should always (1) start with a small and simple solution, (2) design at a high level (`docs/design.md`) before implementation, and (3) frequently ask humans for feedback and clarification. 15 | > {: .warning } 16 | 17 | ## Agentic Coding Steps 18 | 19 | Agentic Coding should be a collaboration between Human System Design and Agent Implementation: 20 | 21 | | Steps | Human | AI | Comment | 22 | | :---------------- | :--------: | :--------: | :--------------------------------------------------------------------------------------------- | 23 | | 1. Requirements | ★★★ High | ★☆☆ Low | Humans understand the requirements and context. | 24 | | 2. Flow | ★★☆ Medium | ★★☆ Medium | Humans specify the high-level design, and the AI fills in the details. | 25 | | 3. Utilities | ★★☆ Medium | ★★☆ Medium | Humans provide available external APIs and integrations, and the AI helps with implementation. | 26 | | 4. Node | ★☆☆ Low | ★★★ High | The AI helps design the node types and data handling based on the flow. | 27 | | 5. Implementation | ★☆☆ Low | ★★★ High | The AI implements the flow based on the design. | 28 | | 6. Optimization | ★★☆ Medium | ★★☆ Medium | Humans evaluate the results, and the AI helps optimize. | 29 | | 7. Reliability | ★☆☆ Low | ★★★ High | The AI writes test cases and addresses corner cases. | 30 | 31 | 1. **Requirements**: Clarify the requirements for your project, and evaluate whether an AI system is a good fit. 32 | 33 | - Understand AI systems' strengths and limitations: 34 | - **Good for**: Routine tasks requiring common sense (filling forms, replying to emails) 35 | - **Good for**: Creative tasks with well-defined inputs (building slides, writing SQL) 36 | - **Not good for**: Ambiguous problems requiring complex decision-making (business strategy, startup planning) 37 | - **Keep It User-Centric:** Explain the "problem" from the user's perspective rather than just listing features. 38 | - **Balance complexity vs. impact**: Aim to deliver the highest value features with minimal complexity early. 39 | 40 | 2. **Flow Design**: Outline at a high level, describe how your AI system orchestrates nodes. 41 | 42 | - Identify applicable design patterns (e.g., [Map Reduce](./design_pattern/mapreduce.md), [Agent](./design_pattern/agent.md), [RAG](./design_pattern/rag.md)). 43 | - For each node in the flow, start with a high-level one-line description of what it does. 44 | - If using **Map Reduce**, specify how to map (what to split) and how to reduce (how to combine). 45 | - If using **Agent**, specify what are the inputs (context) and what are the possible actions. 46 | - If using **RAG**, specify what to embed, noting that there's usually both offline (indexing) and online (retrieval) workflows. 47 | - Outline the flow and draw it in a mermaid diagram. For example: 48 | 49 | ```mermaid 50 | flowchart LR 51 | start[Start] --> batch[Batch] 52 | batch --> check[Check] 53 | check -->|OK| process 54 | check -->|Error| fix[Fix] 55 | fix --> check 56 | 57 | subgraph process[Process] 58 | step1[Step 1] --> step2[Step 2] 59 | end 60 | 61 | process --> endNode[End] 62 | ``` 63 | 64 | - > **If Humans can't specify the flow, AI Agents can't automate it!** Before building an LLM system, thoroughly understand the problem and potential solution by manually solving example inputs to develop intuition. 65 | > {: .best-practice } 66 | 67 | 3. **Utilities**: Based on the Flow Design, identify and implement necessary utility functions. 68 | 69 | - Think of your AI system as the brain. It needs a body—these _external utility functions_—to interact with the real world: 70 |
71 | 72 | - Reading inputs (e.g., retrieving Slack messages, reading emails) 73 | - Writing outputs (e.g., generating reports, sending emails) 74 | - Using external tools (e.g., calling LLMs, searching the web) 75 | - **NOTE**: _LLM-based tasks_ (e.g., summarizing text, analyzing sentiment) are **NOT** utility functions; rather, they are _core functions_ internal in the AI system. 76 | 77 | - For each utility function, implement it and write a simple test. 78 | - Document their input/output, as well as why they are necessary. For example: 79 | - `name`: `getEmbedding` (`src/utils/getEmbedding.ts`) 80 | - `input`: `string` 81 | - `output`: a vector of 3072 numbers 82 | - `necessity`: Used by the second node to embed text 83 | - Example utility implementation: 84 | 85 | ```typescript 86 | // src/utils/callLlm.ts 87 | import { OpenAI } from "openai"; 88 | 89 | export async function callLlm(prompt: string): Promise { 90 | const client = new OpenAI({ 91 | apiKey: process.env.OPENAI_API_KEY, 92 | }); 93 | 94 | const response = await client.chat.completions.create({ 95 | model: "gpt-4o", 96 | messages: [{ role: "user", content: prompt }], 97 | }); 98 | 99 | return response.choices[0].message.content || ""; 100 | } 101 | ``` 102 | 103 | - > **Sometimes, design Utilies before Flow:** For example, for an LLM project to automate a legacy system, the bottleneck will likely be the available interface to that system. Start by designing the hardest utilities for interfacing, and then build the flow around them. 104 | > {: .best-practice } 105 | 106 | 4. **Node Design**: Plan how each node will read and write data, and use utility functions. 107 | 108 | - One core design principle for PocketFlow is to use a [shared store](./core_abstraction/communication.md), so start with a shared store design: 109 | 110 | - For simple systems, use an in-memory object. 111 | - For more complex systems or when persistence is required, use a database. 112 | - **Don't Repeat Yourself**: Use in-memory references or foreign keys. 113 | - Example shared store design: 114 | 115 | ```typescript 116 | interface SharedStore { 117 | user: { 118 | id: string; 119 | context: { 120 | weather: { temp: number; condition: string }; 121 | location: string; 122 | }; 123 | }; 124 | results: Record; 125 | } 126 | 127 | const shared: SharedStore = { 128 | user: { 129 | id: "user123", 130 | context: { 131 | weather: { temp: 72, condition: "sunny" }, 132 | location: "San Francisco", 133 | }, 134 | }, 135 | results: {}, // Empty object to store outputs 136 | }; 137 | ``` 138 | 139 | - For each [Node](./core_abstraction/node.md), describe its type, how it reads and writes data, and which utility function it uses. Keep it specific but high-level without codes. For example: 140 | - `type`: Node (or BatchNode, or ParallelBatchNode) 141 | - `prep`: Read "text" from the shared store 142 | - `exec`: Call the embedding utility function 143 | - `post`: Write "embedding" to the shared store 144 | 145 | 5. **Implementation**: Implement the initial nodes and flows based on the design. 146 | 147 | - 🎉 If you've reached this step, humans have finished the design. Now _Agentic Coding_ begins! 148 | - **"Keep it simple, stupid!"** Avoid complex features and full-scale type checking. 149 | - **FAIL FAST**! Avoid `try` logic so you can quickly identify any weak points in the system. 150 | - Add logging throughout the code to facilitate debugging. 151 | 152 | 6. **Optimization**: 153 | 154 | - **Use Intuition**: For a quick initial evaluation, human intuition is often a good start. 155 | - **Redesign Flow (Back to Step 3)**: Consider breaking down tasks further, introducing agentic decisions, or better managing input contexts. 156 | - If your flow design is already solid, move on to micro-optimizations: 157 | 158 | - **Prompt Engineering**: Use clear, specific instructions with examples to reduce ambiguity. 159 | - **In-Context Learning**: Provide robust examples for tasks that are difficult to specify with instructions alone. 160 | 161 | - > **You'll likely iterate a lot!** Expect to repeat Steps 3–6 hundreds of times. 162 | > 163 | >
164 | > {: .best-practice } 165 | 166 | 7. **Reliability** 167 | - **Node Retries**: Add checks in the node `exec` to ensure outputs meet requirements, and consider increasing `maxRetries` and `wait` times. 168 | - **Logging and Visualization**: Maintain logs of all attempts and visualize node results for easier debugging. 169 | - **Self-Evaluation**: Add a separate node (powered by an LLM) to review outputs when results are uncertain. 170 | 171 | ## Example LLM Project File Structure 172 | 173 | ``` 174 | my-project/ 175 | ├── src/ 176 | │ ├── index.ts 177 | │ ├── nodes.ts 178 | │ ├── flow.ts 179 | │ ├── types.ts 180 | │ └── utils/ 181 | │ ├── callLlm.ts 182 | │ └── searchWeb.ts 183 | ├── docs/ 184 | │ └── design.md 185 | ├── package.json 186 | └── tsconfig.json 187 | ``` 188 | 189 | - **`docs/design.md`**: Contains project documentation for each step above. This should be _high-level_ and _no-code_. 190 | - **`src/types.ts`**: Contains shared type definitions and interfaces used throughout the project. 191 | - **`src/utils/`**: Contains all utility functions. 192 | - It's recommended to dedicate one TypeScript file to each API call, for example `callLlm.ts` or `searchWeb.ts`. 193 | - Each file should export functions that can be imported elsewhere in the project 194 | - Include test cases for each utility function using `utilityFunctionName.test.ts` 195 | - **`src/nodes.ts`**: Contains all the node definitions. 196 | 197 | ```typescript 198 | // src/types.ts 199 | export interface QASharedStore { 200 | question?: string; 201 | answer?: string; 202 | } 203 | ``` 204 | 205 | ```typescript 206 | // src/nodes.ts 207 | import { Node } from "pocketflow"; 208 | import { callLlm } from "./utils/callLlm"; 209 | import { QASharedStore } from "./types"; 210 | import PromptSync from "prompt-sync"; 211 | 212 | const prompt = PromptSync(); 213 | 214 | export class GetQuestionNode extends Node { 215 | async exec(): Promise { 216 | // Get question directly from user input 217 | const userQuestion = prompt("Enter your question: ") || ""; 218 | return userQuestion; 219 | } 220 | 221 | async post( 222 | shared: QASharedStore, 223 | _: unknown, 224 | execRes: string 225 | ): Promise { 226 | // Store the user's question 227 | shared.question = execRes; 228 | return "default"; // Go to the next node 229 | } 230 | } 231 | 232 | export class AnswerNode extends Node { 233 | async prep(shared: QASharedStore): Promise { 234 | // Read question from shared 235 | return shared.question || ""; 236 | } 237 | 238 | async exec(question: string): Promise { 239 | // Call LLM to get the answer 240 | return await callLlm(question); 241 | } 242 | 243 | async post( 244 | shared: QASharedStore, 245 | _: unknown, 246 | execRes: string 247 | ): Promise { 248 | // Store the answer in shared 249 | shared.answer = execRes; 250 | return undefined; 251 | } 252 | } 253 | ``` 254 | 255 | - **`src/flow.ts`**: Implements functions that create flows by importing node definitions and connecting them. 256 | 257 | ```typescript 258 | // src/flow.ts 259 | import { Flow } from "pocketflow"; 260 | import { GetQuestionNode, AnswerNode } from "./nodes"; 261 | import { QASharedStore } from "./types"; 262 | 263 | export function createQaFlow(): Flow { 264 | // Create nodes 265 | const getQuestionNode = new GetQuestionNode(); 266 | const answerNode = new AnswerNode(); 267 | 268 | // Connect nodes in sequence 269 | getQuestionNode.next(answerNode); 270 | 271 | // Create flow starting with input node 272 | return new Flow(getQuestionNode); 273 | } 274 | ``` 275 | 276 | - **`src/index.ts`**: Serves as the project's entry point. 277 | 278 | ```typescript 279 | // src/index.ts 280 | import { createQaFlow } from "./flow"; 281 | import { QASharedStore } from "./types"; 282 | 283 | // Example main function 284 | async function main(): Promise { 285 | const shared: QASharedStore = { 286 | question: undefined, // Will be populated by GetQuestionNode from user input 287 | answer: undefined, // Will be populated by AnswerNode 288 | }; 289 | 290 | // Create the flow and run it 291 | const qaFlow = createQaFlow(); 292 | await qaFlow.run(shared); 293 | console.log(`Question: ${shared.question}`); 294 | console.log(`Answer: ${shared.answer}`); 295 | } 296 | 297 | // Run the main function 298 | main().catch(console.error); 299 | ``` 300 | 301 | - **`package.json`**: Contains project metadata and dependencies. 302 | 303 | - **`tsconfig.json`**: Contains TypeScript compiler configuration. 304 | 305 | ================================================ 306 | File: docs/index.md 307 | ================================================ 308 | 309 | --- 310 | 311 | layout: default 312 | title: "Home" 313 | nav_order: 1 314 | 315 | --- 316 | 317 | --- 318 | 319 | layout: default 320 | title: "Home" 321 | nav_order: 1 322 | 323 | --- 324 | 325 | # PocketFlow.js 326 | 327 | A [100-line](https://github.com/The-Pocket/PocketFlow-Typescript/blob/main/src/index.ts) minimalist LLM framework for _Agents, Task Decomposition, RAG, etc_. 328 | 329 | - **Lightweight**: Just the core graph abstraction in 100 lines. ZERO dependencies, and vendor lock-in. 330 | - **Expressive**: Everything you love from larger frameworks—([Multi-](./design_pattern/multi_agent.html))[Agents](./design_pattern/agent.html), [Workflow](./design_pattern/workflow.html), [RAG](./design_pattern/rag.html), and more. 331 | - **Agentic-Coding**: Intuitive enough for AI agents to help humans build complex LLM applications. 332 | 333 |
334 | 335 |
336 | 337 | ## Core Abstraction 338 | 339 | We model the LLM workflow as a **Graph + Shared Store**: 340 | 341 | - [Node](./core_abstraction/node.md) handles simple (LLM) tasks. 342 | - [Flow](./core_abstraction/flow.md) connects nodes through **Actions** (labeled edges). 343 | - [Shared Store](./core_abstraction/communication.md) enables communication between nodes within flows. 344 | - [Batch](./core_abstraction/batch.md) nodes/flows allow for data-intensive tasks. 345 | - [(Advanced) Parallel](./core_abstraction/parallel.md) nodes/flows handle I/O-bound tasks. 346 | 347 |
348 | 349 |
350 | 351 | ## Design Pattern 352 | 353 | From there, it’s easy to implement popular design patterns: 354 | 355 | - [Agent](./design_pattern/agent.md) autonomously makes decisions. 356 | - [Workflow](./design_pattern/workflow.md) chains multiple tasks into pipelines. 357 | - [RAG](./design_pattern/rag.md) integrates data retrieval with generation. 358 | - [Map Reduce](./design_pattern/mapreduce.md) splits data tasks into Map and Reduce steps. 359 | - [Structured Output](./design_pattern/structure.md) formats outputs consistently. 360 | - [(Advanced) Multi-Agents](./design_pattern/multi_agent.md) coordinate multiple agents. 361 | 362 |
363 | 364 |
365 | 366 | ## Utility Function 367 | 368 | We **do not** provide built-in utilities. Instead, we offer _examples_—please _implement your own_: 369 | 370 | - [LLM Wrapper](./utility_function/llm.md) 371 | - [Viz and Debug](./utility_function/viz.md) 372 | - [Web Search](./utility_function/websearch.md) 373 | - [Chunking](./utility_function/chunking.md) 374 | - [Embedding](./utility_function/embedding.md) 375 | - [Vector Databases](./utility_function/vector.md) 376 | - [Text-to-Speech](./utility_function/text_to_speech.md) 377 | 378 | **Why not built-in?**: I believe it's a _bad practice_ for vendor-specific APIs in a general framework: 379 | 380 | - _API Volatility_: Frequent changes lead to heavy maintenance for hardcoded APIs. 381 | - _Flexibility_: You may want to switch vendors, use fine-tuned models, or run them locally. 382 | - _Optimizations_: Prompt caching, batching, and streaming are easier without vendor lock-in. 383 | 384 | ## Ready to build your Apps? 385 | 386 | Check out [Agentic Coding Guidance](./guide.md), the fastest way to develop LLM projects with PocketFlow.js! 387 | 388 | ================================================ 389 | File: docs/core_abstraction/batch.md 390 | ================================================ 391 | 392 | --- 393 | 394 | layout: default 395 | title: "Batch" 396 | parent: "Core Abstraction" 397 | nav_order: 4 398 | 399 | --- 400 | 401 | # Batch 402 | 403 | **Batch** makes it easier to handle large inputs in one Node or **rerun** a Flow multiple times. Example use cases: 404 | 405 | - **Chunk-based** processing (e.g., splitting large texts). 406 | - **Iterative** processing over lists of input items (e.g., user queries, files, URLs). 407 | 408 | ## 1. BatchNode 409 | 410 | A **BatchNode** extends `Node` but changes `prep()` and `exec()`: 411 | 412 | - **`prep(shared)`**: returns an **array** of items to process. 413 | - **`exec(item)`**: called **once** per item in that iterable. 414 | - **`post(shared, prepRes, execResList)`**: after all items are processed, receives a **list** of results (`execResList`) and returns an **Action**. 415 | 416 | ### Example: Summarize a Large File 417 | 418 | ```typescript 419 | type SharedStorage = { 420 | data: string; 421 | summary?: string; 422 | }; 423 | 424 | class MapSummaries extends BatchNode { 425 | async prep(shared: SharedStorage): Promise { 426 | // Chunk content into manageable pieces 427 | const content = shared.data; 428 | const chunks: string[] = []; 429 | const chunkSize = 10000; 430 | 431 | for (let i = 0; i < content.length; i += chunkSize) { 432 | chunks.push(content.slice(i, i + chunkSize)); 433 | } 434 | 435 | return chunks; 436 | } 437 | 438 | async exec(chunk: string): Promise { 439 | const prompt = `Summarize this chunk in 10 words: ${chunk}`; 440 | return await callLlm(prompt); 441 | } 442 | 443 | async post( 444 | shared: SharedStorage, 445 | _: string[], 446 | summaries: string[] 447 | ): Promise { 448 | shared.summary = summaries.join("\n"); 449 | return "default"; 450 | } 451 | } 452 | 453 | // Usage 454 | const flow = new Flow(new MapSummaries()); 455 | await flow.run({ data: "very long text content..." }); 456 | ``` 457 | 458 | --- 459 | 460 | ## 2. BatchFlow 461 | 462 | A **BatchFlow** runs a **Flow** multiple times, each time with different `params`. Think of it as a loop that replays the Flow for each parameter set. 463 | 464 | ### Example: Summarize Many Files 465 | 466 | ```typescript 467 | type SharedStorage = { 468 | files: string[]; 469 | }; 470 | 471 | type FileParams = { 472 | filename: string; 473 | }; 474 | 475 | class SummarizeAllFiles extends BatchFlow { 476 | async prep(shared: SharedStorage): Promise { 477 | return shared.files.map((filename) => ({ filename })); 478 | } 479 | } 480 | 481 | // Create a per-file summarization flow 482 | const summarizeFile = new SummarizeFile(); 483 | const summarizeAllFiles = new SummarizeAllFiles(summarizeFile); 484 | 485 | await summarizeAllFiles.run({ files: ["file1.txt", "file2.txt"] }); 486 | ``` 487 | 488 | ### Under the Hood 489 | 490 | 1. `prep(shared)` returns a list of param objects—e.g., `[{filename: "file1.txt"}, {filename: "file2.txt"}, ...]`. 491 | 2. The **BatchFlow** loops through each object and: 492 | - Merges it with the BatchFlow's own `params` 493 | - Calls `flow.run(shared)` using the merged result 494 | 3. This means the sub-Flow runs **repeatedly**, once for every param object. 495 | 496 | --- 497 | 498 | ## 3. Nested Batches 499 | 500 | You can nest BatchFlows to handle hierarchical data processing: 501 | 502 | ```typescript 503 | type DirectoryParams = { 504 | directory: string; 505 | }; 506 | 507 | type FileParams = DirectoryParams & { 508 | filename: string; 509 | }; 510 | 511 | class FileBatchFlow extends BatchFlow { 512 | async prep(shared: SharedStorage): Promise { 513 | const directory = this._params.directory; 514 | const files = await getFilesInDirectory(directory).filter((f) => 515 | f.endsWith(".txt") 516 | ); 517 | 518 | return files.map((filename) => ({ 519 | directory, // Pass on directory from parent 520 | filename, // Add filename for this batch item 521 | })); 522 | } 523 | } 524 | 525 | class DirectoryBatchFlow extends BatchFlow { 526 | async prep(shared: SharedStorage): Promise { 527 | return ["/path/to/dirA", "/path/to/dirB"].map((directory) => ({ 528 | directory, 529 | })); 530 | } 531 | } 532 | 533 | // Process all files in all directories 534 | const processingNode = new ProcessingNode(); 535 | const fileFlow = new FileBatchFlow(processingNode); 536 | const dirFlow = new DirectoryBatchFlow(fileFlow); 537 | await dirFlow.run({}); 538 | ``` 539 | 540 | ================================================ 541 | File: docs/core_abstraction/communication.md 542 | ================================================ 543 | 544 | --- 545 | 546 | layout: default 547 | title: "Communication" 548 | parent: "Core Abstraction" 549 | nav_order: 3 550 | 551 | --- 552 | 553 | # Communication 554 | 555 | Nodes and Flows **communicate** in 2 ways: 556 | 557 | 1. **Shared Store (for almost all the cases)** 558 | 559 | - A global data structure (often an in-mem dict) that all nodes can read ( `prep()`) and write (`post()`). 560 | - Great for data results, large content, or anything multiple nodes need. 561 | - You shall design the data structure and populate it ahead. 562 | - > **Separation of Concerns:** Use `Shared Store` for almost all cases to separate _Data Schema_ from _Compute Logic_! This approach is both flexible and easy to manage, resulting in more maintainable code. `Params` is more a syntax sugar for [Batch](./batch.md). 563 | > {: .best-practice } 564 | 565 | 2. **Params (only for [Batch](./batch.md))** 566 | - Each node has a local, ephemeral `params` dict passed in by the **parent Flow**, used as an identifier for tasks. Parameter keys and values shall be **immutable**. 567 | - Good for identifiers like filenames or numeric IDs, in Batch mode. 568 | 569 | If you know memory management, think of the **Shared Store** like a **heap** (shared by all function calls), and **Params** like a **stack** (assigned by the caller). 570 | 571 | --- 572 | 573 | ## 1. Shared Store 574 | 575 | ### Overview 576 | 577 | A shared store is typically an in-mem dictionary, like: 578 | 579 | ```typescript 580 | interface SharedStore { 581 | data: Record; 582 | summary: Record; 583 | config: Record; 584 | // ...other properties 585 | } 586 | 587 | const shared: SharedStore = { data: {}, summary: {}, config: {} /* ... */ }; 588 | ``` 589 | 590 | It can also contain local file handlers, DB connections, or a combination for persistence. We recommend deciding the data structure or DB schema first based on your app requirements. 591 | 592 | ### Example 593 | 594 | ```typescript 595 | interface SharedStore { 596 | data: string; 597 | summary: string; 598 | } 599 | 600 | class LoadData extends Node { 601 | async post( 602 | shared: SharedStore, 603 | prepRes: unknown, 604 | execRes: unknown 605 | ): Promise { 606 | // We write data to shared store 607 | shared.data = "Some text content"; 608 | return "default"; 609 | } 610 | } 611 | 612 | class Summarize extends Node { 613 | async prep(shared: SharedStore): Promise { 614 | // We read data from shared store 615 | return shared.data; 616 | } 617 | 618 | async exec(prepRes: unknown): Promise { 619 | // Call LLM to summarize 620 | const prompt = `Summarize: ${prepRes}`; 621 | const summary = await callLlm(prompt); 622 | return summary; 623 | } 624 | 625 | async post( 626 | shared: SharedStore, 627 | prepRes: unknown, 628 | execRes: unknown 629 | ): Promise { 630 | // We write summary to shared store 631 | shared.summary = execRes as string; 632 | return "default"; 633 | } 634 | } 635 | 636 | const loadData = new LoadData(); 637 | const summarize = new Summarize(); 638 | loadData.next(summarize); 639 | const flow = new Flow(loadData); 640 | 641 | const shared: SharedStore = { data: "", summary: "" }; 642 | flow.run(shared); 643 | ``` 644 | 645 | Here: 646 | 647 | - `LoadData` writes to `shared.data`. 648 | - `Summarize` reads from `shared.data`, summarizes, and writes to `shared.summary`. 649 | 650 | --- 651 | 652 | ## 2. Params 653 | 654 | **Params** let you store _per-Node_ or _per-Flow_ config that doesn't need to live in the shared store. They are: 655 | 656 | - **Immutable** during a Node's run cycle (i.e., they don't change mid-`prep->exec->post`). 657 | - **Set** via `setParams()`. 658 | - **Cleared** and updated each time a parent Flow calls it. 659 | 660 | > Only set the uppermost Flow params because others will be overwritten by the parent Flow. 661 | > 662 | > If you need to set child node params, see [Batch](./batch.md). 663 | > {: .warning } 664 | 665 | Typically, **Params** are identifiers (e.g., file name, page number). Use them to fetch the task you assigned or write to a specific part of the shared store. 666 | 667 | ### Example 668 | 669 | ```typescript 670 | interface SharedStore { 671 | data: Record; 672 | summary: Record; 673 | } 674 | 675 | interface SummarizeParams { 676 | filename: string; 677 | } 678 | 679 | // 1) Create a Node that uses params 680 | class SummarizeFile extends Node { 681 | async prep(shared: SharedStore): Promise { 682 | // Access the node's param 683 | const filename = this._params.filename; 684 | return shared.data[filename] || ""; 685 | } 686 | 687 | async exec(prepRes: unknown): Promise { 688 | const prompt = `Summarize: ${prepRes}`; 689 | return await callLlm(prompt); 690 | } 691 | 692 | async post( 693 | shared: SharedStore, 694 | prepRes: unknown, 695 | execRes: unknown 696 | ): Promise { 697 | const filename = this._params.filename; 698 | shared.summary[filename] = execRes as string; 699 | return "default"; 700 | } 701 | } 702 | 703 | // 2) Set params 704 | const node = new SummarizeFile(); 705 | 706 | // 3) Set Node params directly (for testing) 707 | node.setParams({ filename: "doc1.txt" }); 708 | node.run(shared); 709 | 710 | // 4) Create Flow 711 | const flow = new Flow(node); 712 | 713 | // 5) Set Flow params (overwrites node params) 714 | flow.setParams({ filename: "doc2.txt" }); 715 | flow.run(shared); // The node summarizes doc2, not doc1 716 | ``` 717 | 718 | ================================================ 719 | File: docs/core_abstraction/flow.md 720 | ================================================ 721 | 722 | --- 723 | 724 | layout: default 725 | title: "Flow" 726 | parent: "Core Abstraction" 727 | nav_order: 2 728 | 729 | --- 730 | 731 | # Flow 732 | 733 | A **Flow** orchestrates a graph of Nodes. You can chain Nodes in a sequence or create branching depending on the **Actions** returned from each Node's `post()`. 734 | 735 | ## 1. Action-based Transitions 736 | 737 | Each Node's `post()` returns an **Action** string. By default, if `post()` doesn't return anything, we treat that as `"default"`. 738 | 739 | You define transitions with the syntax: 740 | 741 | 1. **Basic default transition**: `nodeA.next(nodeB)` 742 | This means if `nodeA.post()` returns `"default"`, go to `nodeB`. 743 | (Equivalent to `nodeA.on("default", nodeB)`) 744 | 745 | 2. **Named action transition**: `nodeA.on("actionName", nodeB)` 746 | This means if `nodeA.post()` returns `"actionName"`, go to `nodeB`. 747 | 748 | It's possible to create loops, branching, or multi-step flows. 749 | 750 | ## 2. Method Chaining 751 | 752 | The transition methods support **chaining** for more concise flow creation: 753 | 754 | ### Chaining `on()` Methods 755 | 756 | The `on()` method returns the current node, so you can chain multiple action definitions: 757 | 758 | ```typescript 759 | // All transitions from the same node 760 | nodeA.on("approved", nodeB).on("rejected", nodeC).on("needs_review", nodeD); 761 | 762 | // Equivalent to: 763 | nodeA.on("approved", nodeB); 764 | nodeA.on("rejected", nodeC); 765 | nodeA.on("needs_review", nodeD); 766 | ``` 767 | 768 | ### Chaining `next()` Methods 769 | 770 | The `next()` method returns the target node, allowing you to create linear sequences in a single expression: 771 | 772 | ```typescript 773 | // Creates a linear A → B → C → D sequence 774 | nodeA.next(nodeB).next(nodeC).next(nodeD); 775 | 776 | // Equivalent to: 777 | nodeA.next(nodeB); 778 | nodeB.next(nodeC); 779 | nodeC.next(nodeD); 780 | ``` 781 | 782 | ### Combining Chain Types 783 | 784 | You can combine both chaining styles for complex flows: 785 | 786 | ```typescript 787 | nodeA 788 | .on("action1", nodeB.next(nodeC).next(nodeD)) 789 | .on("action2", nodeE.on("success", nodeF).on("failure", nodeG)); 790 | ``` 791 | 792 | ## 3. Creating a Flow 793 | 794 | A **Flow** begins with a **start** node. You call `const flow = new Flow(someNode)` to specify the entry point. When you call `flow.run(shared)`, it executes the start node, looks at its returned Action from `post()`, follows the transition, and continues until there's no next node. 795 | 796 | ### Example: Simple Sequence 797 | 798 | Here's a minimal flow of two nodes in a chain: 799 | 800 | ```typescript 801 | nodeA.next(nodeB); 802 | const flow = new Flow(nodeA); 803 | flow.run(shared); 804 | ``` 805 | 806 | - When you run the flow, it executes `nodeA`. 807 | - Suppose `nodeA.post()` returns `"default"`. 808 | - The flow then sees `"default"` Action is linked to `nodeB` and runs `nodeB`. 809 | - `nodeB.post()` returns `"default"` but we didn't define a successor for `nodeB`. So the flow ends there. 810 | 811 | ### Example: Branching & Looping 812 | 813 | Here's a simple expense approval flow that demonstrates branching and looping. The `ReviewExpense` node can return three possible Actions: 814 | 815 | - `"approved"`: expense is approved, move to payment processing 816 | - `"needs_revision"`: expense needs changes, send back for revision 817 | - `"rejected"`: expense is denied, finish the process 818 | 819 | We can wire them like this: 820 | 821 | ```typescript 822 | // Define the flow connections 823 | review.on("approved", payment); // If approved, process payment 824 | review.on("needs_revision", revise); // If needs changes, go to revision 825 | review.on("rejected", finish); // If rejected, finish the process 826 | 827 | revise.next(review); // After revision, go back for another review 828 | payment.next(finish); // After payment, finish the process 829 | 830 | const flow = new Flow(review); 831 | ``` 832 | 833 | Let's see how it flows: 834 | 835 | 1. If `review.post()` returns `"approved"`, the expense moves to the `payment` node 836 | 2. If `review.post()` returns `"needs_revision"`, it goes to the `revise` node, which then loops back to `review` 837 | 3. If `review.post()` returns `"rejected"`, it moves to the `finish` node and stops 838 | 839 | ```mermaid 840 | flowchart TD 841 | review[Review Expense] -->|approved| payment[Process Payment] 842 | review -->|needs_revision| revise[Revise Report] 843 | review -->|rejected| finish[Finish Process] 844 | 845 | revise --> review 846 | payment --> finish 847 | ``` 848 | 849 | ### Running Individual Nodes vs. Running a Flow 850 | 851 | - `node.run(shared)`: Just runs that node alone (calls `prep->exec->post()`), returns an Action. 852 | - `flow.run(shared)`: Executes from the start node, follows Actions to the next node, and so on until the flow can't continue. 853 | 854 | > `node.run(shared)` **does not** proceed to the successor. 855 | > This is mainly for debugging or testing a single node. 856 | > 857 | > Always use `flow.run(...)` in production to ensure the full pipeline runs correctly. 858 | > {: .warning } 859 | 860 | ## 4. Nested Flows 861 | 862 | A **Flow** can act like a Node, which enables powerful composition patterns. This means you can: 863 | 864 | 1. Use a Flow as a Node within another Flow's transitions. 865 | 2. Combine multiple smaller Flows into a larger Flow for reuse. 866 | 3. Node `params` will be a merging of **all** parents' `params`. 867 | 868 | ### Flow's Node Methods 869 | 870 | A **Flow** is also a **Node**, so it will run `prep()` and `post()`. However: 871 | 872 | - It **won't** run `exec()`, as its main logic is to orchestrate its nodes. 873 | - `post()` always receives `undefined` for `execRes` and should instead get the flow execution results from the shared store. 874 | 875 | ### Basic Flow Nesting 876 | 877 | Here's how to connect a flow to another node: 878 | 879 | ```typescript 880 | // Create a sub-flow 881 | nodeA.next(nodeB); 882 | const subflow = new Flow(nodeA); 883 | 884 | // Connect it to another node 885 | subflow.next(nodeC); 886 | 887 | // Create the parent flow 888 | const parentFlow = new Flow(subflow); 889 | ``` 890 | 891 | When `parentFlow.run()` executes: 892 | 893 | 1. It starts `subflow` 894 | 2. `subflow` runs through its nodes (`nodeA->nodeB`) 895 | 3. After `subflow` completes, execution continues to `nodeC` 896 | 897 | ### Example: Order Processing Pipeline 898 | 899 | Here's a practical example that breaks down order processing into nested flows: 900 | 901 | ```typescript 902 | // Payment processing sub-flow 903 | validatePayment.next(processPayment).next(paymentConfirmation); 904 | const paymentFlow = new Flow(validatePayment); 905 | 906 | // Inventory sub-flow 907 | checkStock.next(reserveItems).next(updateInventory); 908 | const inventoryFlow = new Flow(checkStock); 909 | 910 | // Shipping sub-flow 911 | createLabel.next(assignCarrier).next(schedulePickup); 912 | const shippingFlow = new Flow(createLabel); 913 | 914 | // Connect the flows into a main order pipeline 915 | paymentFlow.next(inventoryFlow).next(shippingFlow); 916 | 917 | // Create the master flow 918 | const orderPipeline = new Flow(paymentFlow); 919 | 920 | // Run the entire pipeline 921 | orderPipeline.run(sharedData); 922 | ``` 923 | 924 | This creates a clean separation of concerns while maintaining a clear execution path: 925 | 926 | ```mermaid 927 | flowchart LR 928 | subgraph order_pipeline[Order Pipeline] 929 | subgraph paymentFlow["Payment Flow"] 930 | A[Validate Payment] --> B[Process Payment] --> C[Payment Confirmation] 931 | end 932 | 933 | subgraph inventoryFlow["Inventory Flow"] 934 | D[Check Stock] --> E[Reserve Items] --> F[Update Inventory] 935 | end 936 | 937 | subgraph shippingFlow["Shipping Flow"] 938 | G[Create Label] --> H[Assign Carrier] --> I[Schedule Pickup] 939 | end 940 | 941 | paymentFlow --> inventoryFlow 942 | inventoryFlow --> shippingFlow 943 | end 944 | ``` 945 | 946 | ================================================ 947 | File: docs/core_abstraction/node.md 948 | ================================================ 949 | 950 | --- 951 | 952 | layout: default 953 | title: "Node" 954 | parent: "Core Abstraction" 955 | nav_order: 1 956 | 957 | --- 958 | 959 | # Node 960 | 961 | A **Node** is the smallest building block. Each Node has 3 steps `prep->exec->post`: 962 | 963 |
964 | 965 |
966 | 967 | 1. `prep(shared)` 968 | 969 | - **Read and preprocess data** from `shared` store. 970 | - Examples: _query DB, read files, or serialize data into a string_. 971 | - Return `prepRes`, which is used by `exec()` and `post()`. 972 | 973 | 2. `exec(prepRes)` 974 | 975 | - **Execute compute logic**, with optional retries and error handling (below). 976 | - Examples: _(mostly) LLM calls, remote APIs, tool use_. 977 | - ⚠️ This shall be only for compute and **NOT** access `shared`. 978 | - ⚠️ If retries enabled, ensure idempotent implementation. 979 | - Return `execRes`, which is passed to `post()`. 980 | 981 | 3. `post(shared, prepRes, execRes)` 982 | - **Postprocess and write data** back to `shared`. 983 | - Examples: _update DB, change states, log results_. 984 | - **Decide the next action** by returning a _string_ (`action = "default"` if _None_). 985 | 986 | > **Why 3 steps?** To enforce the principle of _separation of concerns_. The data storage and data processing are operated separately. 987 | > 988 | > All steps are _optional_. E.g., you can only implement `prep` and `post` if you just need to process data. 989 | > {: .note } 990 | 991 | ### Fault Tolerance & Retries 992 | 993 | You can **retry** `exec()` if it raises an exception via two parameters when define the Node: 994 | 995 | - `max_retries` (int): Max times to run `exec()`. The default is `1` (**no** retry). 996 | - `wait` (int): The time to wait (in **seconds**) before next retry. By default, `wait=0` (no waiting). 997 | `wait` is helpful when you encounter rate-limits or quota errors from your LLM provider and need to back off. 998 | 999 | ```typescript 1000 | const myNode = new SummarizeFile(3, 10); // maxRetries = 3, wait = 10 seconds 1001 | ``` 1002 | 1003 | When an exception occurs in `exec()`, the Node automatically retries until: 1004 | 1005 | - It either succeeds, or 1006 | - The Node has retried `maxRetries - 1` times already and fails on the last attempt. 1007 | 1008 | You can get the current retry times (0-based) from `this.currentRetry`. 1009 | 1010 | ### Graceful Fallback 1011 | 1012 | To **gracefully handle** the exception (after all retries) rather than raising it, override: 1013 | 1014 | ```typescript 1015 | execFallback(prepRes: unknown, error: Error): unknown { 1016 | return "There was an error processing your request."; 1017 | } 1018 | ``` 1019 | 1020 | By default, it just re-raises the exception. 1021 | 1022 | ### Example: Summarize file 1023 | 1024 | ```typescript 1025 | type SharedStore = { 1026 | data: string; 1027 | summary?: string; 1028 | }; 1029 | 1030 | class SummarizeFile extends Node { 1031 | prep(shared: SharedStore): string { 1032 | return shared.data; 1033 | } 1034 | 1035 | exec(content: string): string { 1036 | if (!content) return "Empty file content"; 1037 | 1038 | const prompt = `Summarize this text in 10 words: ${content}`; 1039 | return callLlm(prompt); 1040 | } 1041 | 1042 | execFallback(_: string, error: Error): string { 1043 | return "There was an error processing your request."; 1044 | } 1045 | 1046 | post(shared: SharedStore, _: string, summary: string): string | undefined { 1047 | shared.summary = summary; 1048 | return undefined; // "default" action 1049 | } 1050 | } 1051 | 1052 | // Example usage 1053 | const node = new SummarizeFile(3); // maxRetries = 3 1054 | const shared: SharedStore = { data: "Long text to summarize..." }; 1055 | const action = node.run(shared); 1056 | 1057 | console.log("Action:", action); 1058 | console.log("Summary:", shared.summary); 1059 | ``` 1060 | 1061 | ================================================ 1062 | File: docs/core_abstraction/parallel.md 1063 | ================================================ 1064 | 1065 | --- 1066 | 1067 | layout: default 1068 | title: "(Advanced) Parallel" 1069 | parent: "Core Abstraction" 1070 | nav_order: 6 1071 | 1072 | --- 1073 | 1074 | # (Advanced) Parallel 1075 | 1076 | **Parallel** Nodes and Flows let you run multiple operations **concurrently**—for example, summarizing multiple texts at once. This can improve performance by overlapping I/O and compute. 1077 | 1078 | > Parallel nodes and flows excel at overlapping I/O-bound work—like LLM calls, database queries, API requests, or file I/O. TypeScript's Promise-based implementation allows for truly concurrent execution of asynchronous operations. 1079 | > {: .warning } 1080 | 1081 | > - **Ensure Tasks Are Independent**: If each item depends on the output of a previous item, **do not** parallelize. 1082 | > 1083 | > - **Beware of Rate Limits**: Parallel calls can **quickly** trigger rate limits on LLM services. You may need a **throttling** mechanism. 1084 | > 1085 | > - **Consider Single-Node Batch APIs**: Some LLMs offer a **batch inference** API where you can send multiple prompts in a single call. This is more complex to implement but can be more efficient than launching many parallel requests and mitigates rate limits. 1086 | > {: .best-practice } 1087 | 1088 | ## ParallelBatchNode 1089 | 1090 | Like **BatchNode**, but runs operations in **parallel** using Promise.all(): 1091 | 1092 | ```typescript 1093 | class TextSummarizer extends ParallelBatchNode { 1094 | async prep(shared: SharedStorage): Promise { 1095 | // e.g., multiple texts 1096 | return shared.texts || []; 1097 | } 1098 | 1099 | async exec(text: string): Promise { 1100 | const prompt = `Summarize: ${text}`; 1101 | return await callLlm(prompt); 1102 | } 1103 | 1104 | async post( 1105 | shared: SharedStorage, 1106 | prepRes: string[], 1107 | execRes: string[] 1108 | ): Promise { 1109 | shared.summaries = execRes; 1110 | return "default"; 1111 | } 1112 | } 1113 | 1114 | const node = new TextSummarizer(); 1115 | const flow = new Flow(node); 1116 | ``` 1117 | 1118 | ## ParallelBatchFlow 1119 | 1120 | Parallel version of **BatchFlow**. Each iteration of the sub-flow runs **concurrently** using Promise.all(): 1121 | 1122 | ```typescript 1123 | class SummarizeMultipleFiles extends ParallelBatchFlow { 1124 | async prep(shared: SharedStorage): Promise[]> { 1125 | return (shared.files || []).map((f) => ({ filename: f })); 1126 | } 1127 | } 1128 | 1129 | const subFlow = new Flow(new LoadAndSummarizeFile()); 1130 | const parallelFlow = new SummarizeMultipleFiles(subFlow); 1131 | await parallelFlow.run(shared); 1132 | ``` 1133 | 1134 | ================================================ 1135 | File: docs/design_pattern/agent.md 1136 | ================================================ 1137 | 1138 | --- 1139 | 1140 | layout: default 1141 | title: "Agent" 1142 | parent: "Design Pattern" 1143 | nav_order: 1 1144 | 1145 | --- 1146 | 1147 | # Agent 1148 | 1149 | Agent is a powerful design pattern in which nodes can take dynamic actions based on the context. 1150 | 1151 |
1152 | 1153 |
1154 | 1155 | ## Implement Agent with Graph 1156 | 1157 | 1. **Context and Action:** Implement nodes that supply context and perform actions. 1158 | 2. **Branching:** Use branching to connect each action node to an agent node. Use action to allow the agent to direct the [flow](../core_abstraction/flow.md) between nodes—and potentially loop back for multi-step. 1159 | 3. **Agent Node:** Provide a prompt to decide action—for example: 1160 | 1161 | ```typescript 1162 | ` 1163 | ### CONTEXT 1164 | Task: ${task} 1165 | Previous Actions: ${prevActions} 1166 | Current State: ${state} 1167 | 1168 | ### ACTION SPACE 1169 | [1] search 1170 | Description: Use web search to get results 1171 | Parameters: query (str) 1172 | 1173 | [2] answer 1174 | Description: Conclude based on the results 1175 | Parameters: result (str) 1176 | 1177 | ### NEXT ACTION 1178 | Decide the next action based on the current context. 1179 | Return your response in YAML format: 1180 | 1181 | \`\`\`yaml 1182 | thinking: 1183 | action: 1184 | parameters: 1185 | \`\`\``; 1186 | ``` 1187 | 1188 | The core of building **high-performance** and **reliable** agents boils down to: 1189 | 1190 | 1. **Context Management:** Provide _relevant, minimal context._ For example, rather than including an entire chat history, retrieve the most relevant via [RAG](./rag.md). 1191 | 1192 | 2. **Action Space:** Provide _a well-structured and unambiguous_ set of actions—avoiding overlap like separate `read_databases` or `read_csvs`. 1193 | 1194 | ## Example Good Action Design 1195 | 1196 | - **Incremental:** Feed content in manageable chunks instead of all at once. 1197 | - **Overview-zoom-in:** First provide high-level structure, then allow drilling into details. 1198 | - **Parameterized/Programmable:** Enable parameterized or programmable actions. 1199 | - **Backtracking:** Let the agent undo the last step instead of restarting entirely. 1200 | 1201 | ## Example: Search Agent 1202 | 1203 | This agent: 1204 | 1205 | 1. Decides whether to search or answer 1206 | 2. If searches, loops back to decide if more search needed 1207 | 3. Answers when enough context gathered 1208 | 1209 | ````typescript 1210 | interface SharedState { 1211 | query?: string; 1212 | context?: Array<{ term: string; result: string }>; 1213 | search_term?: string; 1214 | answer?: string; 1215 | } 1216 | 1217 | class DecideAction extends Node { 1218 | async prep(shared: SharedState): Promise<[string, string]> { 1219 | const context = shared.context 1220 | ? JSON.stringify(shared.context) 1221 | : "No previous search"; 1222 | return [shared.query || "", context]; 1223 | } 1224 | 1225 | async exec([query, context]: [string, string]): Promise { 1226 | const prompt = ` 1227 | Given input: ${query} 1228 | Previous search results: ${context} 1229 | Should I: 1) Search web for more info 2) Answer with current knowledge 1230 | Output in yaml: 1231 | \`\`\`yaml 1232 | action: search/answer 1233 | reason: why this action 1234 | search_term: search phrase if action is search 1235 | \`\`\``; 1236 | const resp = await callLlm(prompt); 1237 | const yamlStr = resp.split("```yaml")[1].split("```")[0].trim(); 1238 | return yaml.load(yamlStr); 1239 | } 1240 | 1241 | async post( 1242 | shared: SharedState, 1243 | _: [string, string], 1244 | result: any 1245 | ): Promise { 1246 | if (result.action === "search") { 1247 | shared.search_term = result.search_term; 1248 | } 1249 | return result.action; 1250 | } 1251 | } 1252 | 1253 | class SearchWeb extends Node { 1254 | async prep(shared: SharedState): Promise { 1255 | return shared.search_term || ""; 1256 | } 1257 | 1258 | async exec(searchTerm: string): Promise { 1259 | return await searchWeb(searchTerm); 1260 | } 1261 | 1262 | async post(shared: SharedState, _: string, execRes: string): Promise { 1263 | shared.context = [ 1264 | ...(shared.context || []), 1265 | { term: shared.search_term || "", result: execRes }, 1266 | ]; 1267 | return "decide"; 1268 | } 1269 | } 1270 | 1271 | class DirectAnswer extends Node { 1272 | async prep(shared: SharedState): Promise<[string, string]> { 1273 | return [ 1274 | shared.query || "", 1275 | shared.context ? JSON.stringify(shared.context) : "", 1276 | ]; 1277 | } 1278 | 1279 | async exec([query, context]: [string, string]): Promise { 1280 | return await callLlm(`Context: ${context}\nAnswer: ${query}`); 1281 | } 1282 | 1283 | async post( 1284 | shared: SharedState, 1285 | _: [string, string], 1286 | execRes: string 1287 | ): Promise { 1288 | shared.answer = execRes; 1289 | return undefined; 1290 | } 1291 | } 1292 | 1293 | // Connect nodes 1294 | const decide = new DecideAction(); 1295 | const search = new SearchWeb(); 1296 | const answer = new DirectAnswer(); 1297 | 1298 | decide.on("search", search); 1299 | decide.on("answer", answer); 1300 | search.on("decide", decide); // Loop back 1301 | 1302 | const flow = new Flow(decide); 1303 | await flow.run({ query: "Who won the Nobel Prize in Physics 2024?" }); 1304 | ```` 1305 | 1306 | ================================================ 1307 | File: docs/design_pattern/mapreduce.md 1308 | ================================================ 1309 | 1310 | --- 1311 | 1312 | layout: default 1313 | title: "Map Reduce" 1314 | parent: "Design Pattern" 1315 | nav_order: 4 1316 | 1317 | --- 1318 | 1319 | # Map Reduce 1320 | 1321 | MapReduce is a design pattern suitable when you have either: 1322 | 1323 | - Large input data (e.g., multiple files to process), or 1324 | - Large output data (e.g., multiple forms to fill) 1325 | 1326 | and there is a logical way to break the task into smaller, ideally independent parts. 1327 | 1328 |
1329 | 1330 |
1331 | 1332 | You first break down the task using [BatchNode](../core_abstraction/batch.md) in the map phase, followed by aggregation in the reduce phase. 1333 | 1334 | ### Example: Document Summarization 1335 | 1336 | ```typescript 1337 | type SharedStorage = { 1338 | files?: Record; 1339 | file_summaries?: Record; 1340 | all_files_summary?: string; 1341 | }; 1342 | 1343 | class SummarizeAllFiles extends BatchNode { 1344 | async prep(shared: SharedStorage): Promise<[string, string][]> { 1345 | return Object.entries(shared.files || {}); // [["file1.txt", "aaa..."], ["file2.txt", "bbb..."], ...] 1346 | } 1347 | 1348 | async exec([filename, content]: [string, string]): Promise<[string, string]> { 1349 | const summary = await callLLM(`Summarize the following file:\n${content}`); 1350 | return [filename, summary]; 1351 | } 1352 | 1353 | async post( 1354 | shared: SharedStorage, 1355 | _: [string, string][], 1356 | summaries: [string, string][] 1357 | ): Promise { 1358 | shared.file_summaries = Object.fromEntries(summaries); 1359 | return "summarized"; 1360 | } 1361 | } 1362 | 1363 | class CombineSummaries extends Node { 1364 | async prep(shared: SharedStorage): Promise> { 1365 | return shared.file_summaries || {}; 1366 | } 1367 | 1368 | async exec(summaries: Record): Promise { 1369 | const text_list = Object.entries(summaries).map( 1370 | ([fname, summ]) => `${fname} summary:\n${summ}\n` 1371 | ); 1372 | 1373 | return await callLLM( 1374 | `Combine these file summaries into one final summary:\n${text_list.join( 1375 | "\n---\n" 1376 | )}` 1377 | ); 1378 | } 1379 | 1380 | async post( 1381 | shared: SharedStorage, 1382 | _: Record, 1383 | finalSummary: string 1384 | ): Promise { 1385 | shared.all_files_summary = finalSummary; 1386 | return "combined"; 1387 | } 1388 | } 1389 | 1390 | // Create and connect flow 1391 | const batchNode = new SummarizeAllFiles(); 1392 | const combineNode = new CombineSummaries(); 1393 | batchNode.on("summarized", combineNode); 1394 | 1395 | // Run the flow with test data 1396 | const flow = new Flow(batchNode); 1397 | flow.run({ 1398 | files: { 1399 | "file1.txt": 1400 | "Alice was beginning to get very tired of sitting by her sister...", 1401 | "file2.txt": "Some other interesting text ...", 1402 | }, 1403 | }); 1404 | ``` 1405 | 1406 | > **Performance Tip**: The example above works sequentially. You can speed up the map phase by using `ParallelBatchNode` instead of `BatchNode`. See [(Advanced) Parallel](../core_abstraction/parallel.md) for more details. 1407 | > {: .note } 1408 | 1409 | ================================================ 1410 | File: docs/design_pattern/rag.md 1411 | ================================================ 1412 | 1413 | --- 1414 | 1415 | layout: default 1416 | title: "RAG" 1417 | parent: "Design Pattern" 1418 | nav_order: 3 1419 | 1420 | --- 1421 | 1422 | # RAG (Retrieval Augmented Generation) 1423 | 1424 | For certain LLM tasks like answering questions, providing relevant context is essential. One common architecture is a **two-stage** RAG pipeline: 1425 | 1426 |
1427 | 1428 |
1429 | 1430 | 1. **Offline stage**: Preprocess and index documents ("building the index"). 1431 | 2. **Online stage**: Given a question, generate answers by retrieving the most relevant context. 1432 | 1433 | --- 1434 | 1435 | ## Stage 1: Offline Indexing 1436 | 1437 | We create three Nodes: 1438 | 1439 | 1. `ChunkDocs` – [chunks](../utility_function/chunking.md) raw text. 1440 | 2. `EmbedDocs` – [embeds](../utility_function/embedding.md) each chunk. 1441 | 3. `StoreIndex` – stores embeddings into a [vector database](../utility_function/vector.md). 1442 | 1443 | ```typescript 1444 | type SharedStore = { 1445 | files?: string[]; 1446 | allChunks?: string[]; 1447 | allEmbeds?: number[][]; 1448 | index?: any; 1449 | }; 1450 | 1451 | class ChunkDocs extends BatchNode { 1452 | async prep(shared: SharedStore): Promise { 1453 | return shared.files || []; 1454 | } 1455 | 1456 | async exec(filepath: string): Promise { 1457 | const text = fs.readFileSync(filepath, "utf-8"); 1458 | // Simplified chunking for example 1459 | const chunks: string[] = []; 1460 | const size = 100; 1461 | for (let i = 0; i < text.length; i += size) { 1462 | chunks.push(text.substring(i, i + size)); 1463 | } 1464 | return chunks; 1465 | } 1466 | 1467 | async post( 1468 | shared: SharedStore, 1469 | _: string[], 1470 | chunks: string[][] 1471 | ): Promise { 1472 | shared.allChunks = chunks.flat(); 1473 | return undefined; 1474 | } 1475 | } 1476 | 1477 | class EmbedDocs extends BatchNode { 1478 | async prep(shared: SharedStore): Promise { 1479 | return shared.allChunks || []; 1480 | } 1481 | 1482 | async exec(chunk: string): Promise { 1483 | return await getEmbedding(chunk); 1484 | } 1485 | 1486 | async post( 1487 | shared: SharedStore, 1488 | _: string[], 1489 | embeddings: number[][] 1490 | ): Promise { 1491 | shared.allEmbeds = embeddings; 1492 | return undefined; 1493 | } 1494 | } 1495 | 1496 | class StoreIndex extends Node { 1497 | async prep(shared: SharedStore): Promise { 1498 | return shared.allEmbeds || []; 1499 | } 1500 | 1501 | async exec(allEmbeds: number[][]): Promise { 1502 | return await createIndex(allEmbeds); 1503 | } 1504 | 1505 | async post( 1506 | shared: SharedStore, 1507 | _: number[][], 1508 | index: unknown 1509 | ): Promise { 1510 | shared.index = index; 1511 | return undefined; 1512 | } 1513 | } 1514 | 1515 | // Create indexing flow 1516 | const chunkNode = new ChunkDocs(); 1517 | const embedNode = new EmbedDocs(); 1518 | const storeNode = new StoreIndex(); 1519 | 1520 | chunkNode.next(embedNode).next(storeNode); 1521 | const offlineFlow = new Flow(chunkNode); 1522 | ``` 1523 | 1524 | --- 1525 | 1526 | ## Stage 2: Online Query & Answer 1527 | 1528 | We have 3 nodes: 1529 | 1530 | 1. `EmbedQuery` – embeds the user's question. 1531 | 2. `RetrieveDocs` – retrieves top chunk from the index. 1532 | 3. `GenerateAnswer` – calls the LLM with the question + chunk to produce the final answer. 1533 | 1534 | ```typescript 1535 | type OnlineStore = SharedStore & { 1536 | question?: string; 1537 | qEmb?: number[]; 1538 | retrievedChunk?: string; 1539 | answer?: string; 1540 | }; 1541 | 1542 | class EmbedQuery extends Node { 1543 | async prep(shared: OnlineStore): Promise { 1544 | return shared.question || ""; 1545 | } 1546 | 1547 | async exec(question: string): Promise { 1548 | return await getEmbedding(question); 1549 | } 1550 | 1551 | async post( 1552 | shared: OnlineStore, 1553 | _: string, 1554 | qEmb: number[] 1555 | ): Promise { 1556 | shared.qEmb = qEmb; 1557 | return undefined; 1558 | } 1559 | } 1560 | 1561 | class RetrieveDocs extends Node { 1562 | async prep(shared: OnlineStore): Promise<[number[], any, string[]]> { 1563 | return [shared.qEmb || [], shared.index, shared.allChunks || []]; 1564 | } 1565 | 1566 | async exec([qEmb, index, chunks]: [ 1567 | number[], 1568 | any, 1569 | string[] 1570 | ]): Promise { 1571 | const [ids] = await searchIndex(index, qEmb, { topK: 1 }); 1572 | return chunks[ids[0][0]]; 1573 | } 1574 | 1575 | async post( 1576 | shared: OnlineStore, 1577 | _: [number[], any, string[]], 1578 | chunk: string 1579 | ): Promise { 1580 | shared.retrievedChunk = chunk; 1581 | return undefined; 1582 | } 1583 | } 1584 | 1585 | class GenerateAnswer extends Node { 1586 | async prep(shared: OnlineStore): Promise<[string, string]> { 1587 | return [shared.question || "", shared.retrievedChunk || ""]; 1588 | } 1589 | 1590 | async exec([question, chunk]: [string, string]): Promise { 1591 | return await callLlm(`Question: ${question}\nContext: ${chunk}\nAnswer:`); 1592 | } 1593 | 1594 | async post( 1595 | shared: OnlineStore, 1596 | _: [string, string], 1597 | answer: string 1598 | ): Promise { 1599 | shared.answer = answer; 1600 | return undefined; 1601 | } 1602 | } 1603 | 1604 | // Create query flow 1605 | const embedQNode = new EmbedQuery(); 1606 | const retrieveNode = new RetrieveDocs(); 1607 | const generateNode = new GenerateAnswer(); 1608 | 1609 | embedQNode.next(retrieveNode).next(generateNode); 1610 | const onlineFlow = new Flow(embedQNode); 1611 | ``` 1612 | 1613 | Usage example: 1614 | 1615 | ```typescript 1616 | const shared = { 1617 | files: ["doc1.txt", "doc2.txt"], // any text files 1618 | }; 1619 | await offlineFlow.run(shared); 1620 | ``` 1621 | 1622 | ================================================ 1623 | File: docs/design_pattern/structure.md 1624 | ================================================ 1625 | 1626 | --- 1627 | 1628 | layout: default 1629 | title: "Structured Output" 1630 | parent: "Design Pattern" 1631 | nav_order: 5 1632 | 1633 | --- 1634 | 1635 | # Structured Output 1636 | 1637 | In many use cases, you may want the LLM to output a specific structure, such as a list or a dictionary with predefined keys. 1638 | 1639 | There are several approaches to achieve a structured output: 1640 | 1641 | - **Prompting** the LLM to strictly return a defined structure. 1642 | - Using LLMs that natively support **schema enforcement**. 1643 | - **Post-processing** the LLM's response to extract structured content. 1644 | 1645 | In practice, **Prompting** is simple and reliable for modern LLMs. 1646 | 1647 | ### Example Use Cases 1648 | 1649 | - Extracting Key Information 1650 | 1651 | ```yaml 1652 | product: 1653 | name: Widget Pro 1654 | price: 199.99 1655 | description: | 1656 | A high-quality widget designed for professionals. 1657 | Recommended for advanced users. 1658 | ``` 1659 | 1660 | - Summarizing Documents into Bullet Points 1661 | 1662 | ```yaml 1663 | summary: 1664 | - This product is easy to use. 1665 | - It is cost-effective. 1666 | - Suitable for all skill levels. 1667 | ``` 1668 | 1669 | ## TypeScript Implementation 1670 | 1671 | When using PocketFlow with structured output, follow these TypeScript patterns: 1672 | 1673 | 1. **Define Types** for your structured input/output 1674 | 2. **Implement Validation** in your Node methods 1675 | 3. **Use Type-Safe Operations** throughout your flow 1676 | 1677 | ### Example Text Summarization 1678 | 1679 | ````typescript 1680 | // Define types 1681 | type SummaryResult = { 1682 | summary: string[]; 1683 | }; 1684 | 1685 | type SharedStorage = { 1686 | text?: string; 1687 | result?: SummaryResult; 1688 | }; 1689 | 1690 | class SummarizeNode extends Node { 1691 | async prep(shared: SharedStorage): Promise { 1692 | return shared.text; 1693 | } 1694 | 1695 | async exec(text: string | undefined): Promise { 1696 | if (!text) return { summary: ["No text provided"] }; 1697 | 1698 | const prompt = ` 1699 | Please summarize the following text as YAML, with exactly 3 bullet points 1700 | 1701 | ${text} 1702 | 1703 | Output: 1704 | \`\`\`yaml 1705 | summary: 1706 | - bullet 1 1707 | - bullet 2 1708 | - bullet 3 1709 | \`\`\``; 1710 | 1711 | // Simulated LLM call 1712 | const response = 1713 | "```yaml\nsummary:\n - First point\n - Second insight\n - Final conclusion\n```"; 1714 | 1715 | // Parse YAML response 1716 | const yamlStr = response.split("```yaml")[1].split("```")[0].trim(); 1717 | 1718 | // Extract bullet points 1719 | const result: SummaryResult = { 1720 | summary: yamlStr 1721 | .split("\n") 1722 | .filter((line) => line.trim().startsWith("- ")) 1723 | .map((line) => line.trim().substring(2)), 1724 | }; 1725 | 1726 | // Validate 1727 | if (!result.summary || !Array.isArray(result.summary)) { 1728 | throw new Error("Invalid summary structure"); 1729 | } 1730 | 1731 | return result; 1732 | } 1733 | 1734 | async post( 1735 | shared: SharedStorage, 1736 | _: string | undefined, 1737 | result: SummaryResult 1738 | ): Promise { 1739 | shared.result = result; 1740 | return "default"; 1741 | } 1742 | } 1743 | ```` 1744 | 1745 | ### Why YAML instead of JSON? 1746 | 1747 | Current LLMs struggle with escaping. YAML is easier with strings since they don't always need quotes. 1748 | 1749 | **In JSON** 1750 | 1751 | ```json 1752 | { 1753 | "dialogue": "Alice said: \"Hello Bob.\\nHow are you?\\nI am good.\"" 1754 | } 1755 | ``` 1756 | 1757 | **In YAML** 1758 | 1759 | ```yaml 1760 | dialogue: | 1761 | Alice said: "Hello Bob. 1762 | How are you? 1763 | I am good." 1764 | ``` 1765 | 1766 | ================================================ 1767 | File: docs/design_pattern/workflow.md 1768 | ================================================ 1769 | 1770 | --- 1771 | 1772 | layout: default 1773 | title: "Workflow" 1774 | parent: "Design Pattern" 1775 | nav_order: 2 1776 | 1777 | --- 1778 | 1779 | # Workflow 1780 | 1781 | Many real-world tasks are too complex for one LLM call. The solution is to **Task Decomposition**: decompose them into a [chain](../core_abstraction/flow.md) of multiple Nodes. 1782 | 1783 |
1784 | 1785 |
1786 | 1787 | > - You don't want to make each task **too coarse**, because it may be _too complex for one LLM call_. 1788 | > - You don't want to make each task **too granular**, because then _the LLM call doesn't have enough context_ and results are _not consistent across nodes_. 1789 | > 1790 | > You usually need multiple _iterations_ to find the _sweet spot_. If the task has too many _edge cases_, consider using [Agents](./agent.md). 1791 | > {: .best-practice } 1792 | 1793 | ### Example: Article Writing 1794 | 1795 | ```typescript 1796 | interface SharedState { 1797 | topic?: string; 1798 | outline?: string; 1799 | draft?: string; 1800 | final_article?: string; 1801 | } 1802 | 1803 | // Helper function to simulate LLM call 1804 | async function callLLM(prompt: string): Promise { 1805 | return `Response to: ${prompt}`; 1806 | } 1807 | 1808 | class GenerateOutline extends Node { 1809 | async prep(shared: SharedState): Promise { 1810 | return shared.topic || ""; 1811 | } 1812 | 1813 | async exec(topic: string): Promise { 1814 | return await callLLM( 1815 | `Create a detailed outline for an article about ${topic}` 1816 | ); 1817 | } 1818 | 1819 | async post(shared: SharedState, _: string, outline: string): Promise { 1820 | shared.outline = outline; 1821 | return "default"; 1822 | } 1823 | } 1824 | 1825 | class WriteSection extends Node { 1826 | async prep(shared: SharedState): Promise { 1827 | return shared.outline || ""; 1828 | } 1829 | 1830 | async exec(outline: string): Promise { 1831 | return await callLLM(`Write content based on this outline: ${outline}`); 1832 | } 1833 | 1834 | async post(shared: SharedState, _: string, draft: string): Promise { 1835 | shared.draft = draft; 1836 | return "default"; 1837 | } 1838 | } 1839 | 1840 | class ReviewAndRefine extends Node { 1841 | async prep(shared: SharedState): Promise { 1842 | return shared.draft || ""; 1843 | } 1844 | 1845 | async exec(draft: string): Promise { 1846 | return await callLLM(`Review and improve this draft: ${draft}`); 1847 | } 1848 | 1849 | async post( 1850 | shared: SharedState, 1851 | _: string, 1852 | final: string 1853 | ): Promise { 1854 | shared.final_article = final; 1855 | return undefined; 1856 | } 1857 | } 1858 | 1859 | // Connect nodes in sequence 1860 | const outline = new GenerateOutline(); 1861 | const write = new WriteSection(); 1862 | const review = new ReviewAndRefine(); 1863 | 1864 | outline.next(write).next(review); 1865 | 1866 | // Create and run flow 1867 | const writingFlow = new Flow(outline); 1868 | writingFlow.run({ topic: "AI Safety" }); 1869 | ``` 1870 | 1871 | For _dynamic cases_, consider using [Agents](./agent.md). 1872 | 1873 | ================================================ 1874 | File: docs/utility_function/llm.md 1875 | ================================================ 1876 | 1877 | --- 1878 | 1879 | layout: default 1880 | title: "LLM Wrapper" 1881 | parent: "Utility Function" 1882 | nav_order: 1 1883 | 1884 | --- 1885 | 1886 | # LLM Wrappers 1887 | 1888 | Check out popular libraries like [LangChain](https://github.com/langchain-ai/langchainjs) (13.8k+ GitHub stars), [ModelFusion](https://github.com/vercel/modelfusion) (1.2k+ GitHub stars), or [Firebase GenKit](https://firebase.google.com/docs/genkit) for unified LLM interfaces. 1889 | Here, we provide some minimal example implementations: 1890 | 1891 | 1. OpenAI 1892 | 1893 | ```typescript 1894 | import { OpenAI } from "openai"; 1895 | 1896 | async function callLlm(prompt: string): Promise { 1897 | const client = new OpenAI({ apiKey: "YOUR_API_KEY_HERE" }); 1898 | const r = await client.chat.completions.create({ 1899 | model: "gpt-4o", 1900 | messages: [{ role: "user", content: prompt }], 1901 | }); 1902 | return r.choices[0].message.content || ""; 1903 | } 1904 | 1905 | // Example usage 1906 | callLlm("How are you?").then(console.log); 1907 | ``` 1908 | 1909 | > Store the API key in an environment variable like OPENAI_API_KEY for security. 1910 | > {: .best-practice } 1911 | 1912 | 2. Claude (Anthropic) 1913 | 1914 | ```typescript 1915 | import Anthropic from "@anthropic-ai/sdk"; 1916 | 1917 | async function callLlm(prompt: string): Promise { 1918 | const client = new Anthropic({ 1919 | apiKey: "YOUR_API_KEY_HERE", 1920 | }); 1921 | const response = await client.messages.create({ 1922 | model: "claude-3-7-sonnet-20250219", 1923 | max_tokens: 3000, 1924 | messages: [{ role: "user", content: prompt }], 1925 | }); 1926 | return response.content[0].text; 1927 | } 1928 | ``` 1929 | 1930 | 3. Google (Vertex AI) 1931 | 1932 | ```typescript 1933 | import { VertexAI } from "@google-cloud/vertexai"; 1934 | 1935 | async function callLlm(prompt: string): Promise { 1936 | const vertexAI = new VertexAI({ 1937 | project: "YOUR_PROJECT_ID", 1938 | location: "us-central1", 1939 | }); 1940 | 1941 | const generativeModel = vertexAI.getGenerativeModel({ 1942 | model: "gemini-1.5-flash", 1943 | }); 1944 | 1945 | const response = await generativeModel.generateContent({ 1946 | contents: [{ role: "user", parts: [{ text: prompt }] }], 1947 | }); 1948 | 1949 | return response.response.candidates[0].content.parts[0].text; 1950 | } 1951 | ``` 1952 | 1953 | 4. Azure (Azure OpenAI) 1954 | 1955 | ```typescript 1956 | import { AzureOpenAI } from "openai"; 1957 | 1958 | async function callLlm(prompt: string): Promise { 1959 | const client = new AzureOpenAI({ 1960 | apiKey: "YOUR_API_KEY_HERE", 1961 | azure: { 1962 | apiVersion: "2023-05-15", 1963 | endpoint: "https://.openai.azure.com/", 1964 | }, 1965 | }); 1966 | 1967 | const r = await client.chat.completions.create({ 1968 | model: "", 1969 | messages: [{ role: "user", content: prompt }], 1970 | }); 1971 | 1972 | return r.choices[0].message.content || ""; 1973 | } 1974 | ``` 1975 | 1976 | 5. Ollama (Local LLM) 1977 | 1978 | ```typescript 1979 | import ollama from "ollama"; 1980 | 1981 | async function callLlm(prompt: string): Promise { 1982 | const response = await ollama.chat({ 1983 | model: "llama2", 1984 | messages: [{ role: "user", content: prompt }], 1985 | }); 1986 | return response.message.content; 1987 | } 1988 | ``` 1989 | 1990 | ## Improvements 1991 | 1992 | Feel free to enhance your `callLlm` function as needed. Here are examples: 1993 | 1994 | - Handle chat history: 1995 | 1996 | ```typescript 1997 | interface Message { 1998 | role: "user" | "assistant" | "system"; 1999 | content: string; 2000 | } 2001 | 2002 | async function callLlm(messages: Message[]): Promise { 2003 | const client = new OpenAI({ apiKey: "YOUR_API_KEY_HERE" }); 2004 | const r = await client.chat.completions.create({ 2005 | model: "gpt-4o", 2006 | messages: messages, 2007 | }); 2008 | return r.choices[0].message.content || ""; 2009 | } 2010 | ``` 2011 | 2012 | - Add in-memory caching 2013 | 2014 | ```typescript 2015 | import { memoize } from "lodash"; 2016 | 2017 | const callLlmMemoized = memoize(async (prompt: string): Promise => { 2018 | // Your implementation here 2019 | return ""; 2020 | }); 2021 | 2022 | async function callLlm(prompt: string, useCache = true): Promise { 2023 | if (useCache) { 2024 | return callLlmMemoized(prompt); 2025 | } 2026 | // Call the underlying function directly 2027 | return callLlmInternal(prompt); 2028 | } 2029 | 2030 | class SummarizeNode { 2031 | private curRetry = 0; 2032 | 2033 | async exec(text: string): Promise { 2034 | return callLlm(`Summarize: ${text}`, this.curRetry === 0); 2035 | } 2036 | } 2037 | ``` 2038 | 2039 | - Enable logging: 2040 | 2041 | ```typescript 2042 | async function callLlm(prompt: string): Promise { 2043 | console.info(`Prompt: ${prompt}`); 2044 | // Your implementation here 2045 | const response = ""; // Response from your implementation 2046 | console.info(`Response: ${response}`); 2047 | return response; 2048 | } 2049 | ``` 2050 | --------------------------------------------------------------------------------