├── utils ├── __init__.py ├── call_llm.py ├── search_web.py └── content_retrieval.py ├── assets └── banner.png ├── input.csv ├── requirements.txt ├── LICENSE ├── .gitignore ├── README.md ├── main.py ├── docs └── design.md ├── output.csv ├── main_batch.py ├── flow.py ├── app.py ├── blog.md └── .cursorrules /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/The-Pocket/PocketFlow-Tutorial-Cold-Email-Personalization/master/assets/banner.png -------------------------------------------------------------------------------- /input.csv: -------------------------------------------------------------------------------- 1 | first_name,last_name,keywords 2 | Elon,Musk,Tesla SpaceX entrepreneur 3 | Sundar,Pichai,Google CEO tech 4 | Tim,Cook,Apple iPhone CEO -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | openai>=1.0.0 2 | requests>=2.25.0 3 | pocketflow>=0.0.1 4 | pyyaml>=6.0 5 | beautifulsoup4>=4.9.3 6 | anthropic>=0.12.0 7 | streamlit>=1.24.0 8 | google-api-python-client>=2.0.0 -------------------------------------------------------------------------------- /utils/call_llm.py: -------------------------------------------------------------------------------- 1 | from anthropic import AnthropicVertex 2 | import os 3 | 4 | def call_llm(prompt: str) -> str: 5 | client = AnthropicVertex( 6 | region=os.getenv("ANTHROPIC_REGION", "us-east5"), 7 | project_id=os.getenv("ANTHROPIC_PROJECT_ID", "your-project-id") 8 | ) 9 | response = client.messages.create( 10 | max_tokens=1024, 11 | messages=[{"role": "user", "content": prompt}], 12 | model="claude-3-7-sonnet@20250219" 13 | ) 14 | return response.content[0].text 15 | 16 | if __name__ == "__main__": 17 | test_prompt = "Hello, how are you?" 18 | response = call_llm(test_prompt) 19 | print(f"Test successful. Response: {response}") 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Zachary Huang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | node_modules/ 3 | vendor/ 4 | .pnp/ 5 | .pnp.js 6 | 7 | # Build outputs 8 | dist/ 9 | build/ 10 | out/ 11 | *.pyc 12 | __pycache__/ 13 | 14 | # Environment files 15 | .env 16 | .env.local 17 | .env.*.local 18 | .env.development 19 | .env.test 20 | .env.production 21 | 22 | # IDE - VSCode 23 | .vscode/* 24 | !.vscode/settings.json 25 | !.vscode/tasks.json 26 | !.vscode/launch.json 27 | !.vscode/extensions.json 28 | 29 | # IDE - JetBrains 30 | .idea/ 31 | *.iml 32 | *.iws 33 | *.ipr 34 | 35 | # IDE - Eclipse 36 | .project 37 | .classpath 38 | .settings/ 39 | 40 | # Logs 41 | logs/ 42 | *.log 43 | npm-debug.log* 44 | yarn-debug.log* 45 | yarn-error.log* 46 | 47 | # Operating System 48 | .DS_Store 49 | Thumbs.db 50 | *.swp 51 | *.swo 52 | 53 | # Testing 54 | coverage/ 55 | .nyc_output/ 56 | 57 | # Temporary files 58 | *.tmp 59 | *.temp 60 | .cache/ 61 | 62 | # Compiled files 63 | *.com 64 | *.class 65 | *.dll 66 | *.exe 67 | *.o 68 | *.so 69 | 70 | # Package files 71 | *.7z 72 | *.dmg 73 | *.gz 74 | *.iso 75 | *.jar 76 | *.rar 77 | *.tar 78 | *.zip 79 | 80 | # Database 81 | *.sqlite 82 | *.sqlite3 83 | *.db 84 | 85 | # Optional npm cache directory 86 | .npm 87 | 88 | # Optional eslint cache 89 | .eslintcache 90 | 91 | # Optional REPL history 92 | .node_repl_history -------------------------------------------------------------------------------- /utils/search_web.py: -------------------------------------------------------------------------------- 1 | """ 2 | Web Search Utility for Cold Outreach Opener Generator 3 | """ 4 | import os 5 | import requests 6 | import json 7 | 8 | def search_web(query, num_results=10): 9 | """ 10 | Executes a Google Custom Search and returns results. 11 | :param api_key: Your Google API key 12 | :param cse_id: Your custom search engine ID 13 | :param query: The search query string 14 | :param num_results: Number of search results to return (1-10) 15 | :return: A list of results (each result is a dict with relevant fields) 16 | """ 17 | # Replace these with your actual API key and Search Engine ID. 18 | API_KEY = "google-api-key" 19 | SEARCH_ENGINE_ID = "google-search-engine-id" 20 | 21 | url = "https://www.googleapis.com/customsearch/v1" 22 | params = { 23 | 'key': API_KEY, 24 | 'cx': SEARCH_ENGINE_ID, 25 | 'q': query, 26 | 'num': num_results 27 | } 28 | 29 | response = requests.get(url, params=params) 30 | if response.status_code == 200: 31 | data = response.json() 32 | # Results are typically in data['items'] if the request is successful 33 | return data.get('items', []) 34 | else: 35 | print(f"Error: {response.status_code}, {response.text}") 36 | return [] 37 | 38 | if __name__ == "__main__": 39 | results = search_web("Elon Musk") 40 | 41 | for idx, item in enumerate(results, start=1): 42 | title = item.get("title") 43 | snippet = item.get("snippet") 44 | link = item.get("link") 45 | print(f"{idx}. {title}\n{snippet}\nLink: {link}\n") 46 | -------------------------------------------------------------------------------- /utils/content_retrieval.py: -------------------------------------------------------------------------------- 1 | """ 2 | HTML Content Retrieval Utility for Cold Outreach Opener Generator 3 | """ 4 | import requests 5 | from bs4 import BeautifulSoup 6 | 7 | def get_html_content(url, timeout=10): 8 | """ 9 | Retrieves HTML content from a URL. 10 | 11 | Args: 12 | url (str): URL to retrieve content from 13 | timeout (int, optional): Request timeout in seconds. Defaults to 10. 14 | 15 | Returns: 16 | dict: Dictionary containing HTML content and extracted text 17 | """ 18 | try: 19 | headers = { 20 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' 21 | } 22 | response = requests.get(url, headers=headers, timeout=timeout) 23 | response.raise_for_status() # Raise exception for 4XX/5XX status codes 24 | 25 | html_content = response.text 26 | soup = BeautifulSoup(html_content, 'html.parser') 27 | 28 | # Remove script and style elements 29 | for script in soup(["script", "style"]): 30 | script.extract() 31 | 32 | # Extract text content 33 | text = soup.get_text(separator=' ', strip=True) 34 | 35 | # Clean up text (remove excessive newlines) 36 | lines = (line.strip() for line in text.splitlines()) 37 | text = ' '.join(line for line in lines if line) 38 | 39 | return { 40 | "html": html_content, 41 | "text": text, 42 | "title": soup.title.string if soup.title else "" 43 | } 44 | except Exception as e: 45 | print(f"Error retrieving content from {url}: {e}") 46 | return { 47 | "html": "", 48 | "text": f"Error retrieving content: {str(e)}", 49 | "title": "" 50 | } 51 | 52 | if __name__ == "__main__": 53 | # Test the function 54 | test_url = "https://github.com/The-Pocket/PocketFlow" 55 | content = get_html_content(test_url) 56 | print(f"Title: {content['title']}") 57 | print(f"Text length: {len(content['text'])}") 58 | print("First 200 characters of text:") 59 | print(content['text'][:200] + "...") -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

Cold Outreach Opener Generator

2 | 3 | ![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) 4 | 5 | Doing a cold outreach campaign and want to grab attention from the first sentence? This Open Sourced LLM App instantly crafts personalized openers that boost opens and response rates. 6 | 7 |
8 | 9 |
10 | 11 | - 🔥 [Try Live Demo](https://pocket-opener-851564657364.us-east1.run.app/) 12 | 13 | - Check out the [Blog Post](https://substack.com/home/post/p-159003722) 14 | 15 | ## I built this in just an hour, and you can, too. 16 | 17 | - I built using [**Agentic Coding**](https://the-pocket.github.io/PocketFlow/guide.html), the fastest development paradigm, where humans simply [design](docs/design.md) and agents [code](flow.py). 18 | 19 | - The secret weapon is [Pocket Flow](https://github.com/The-Pocket/PocketFlow), a 100-line LLM framework that lets Agents (e.g., Cursor AI) build for you 20 | 21 | - Step-by-step YouTube development tutorial coming soon! [Subscribe for notifications](https://www.youtube.com/@ZacharyLLM?sub_confirmation=1). 22 | 23 | ## How to Run 24 | 25 | 1. Implement `call_llm` in `utils/call_llm.py`, and `search_web` in `utils/search_web.py`. 26 | 27 | 2. Install the dependencies and run the program: 28 | ```bash 29 | pip install -r requirements.txt 30 | python main.py 31 | ``` 32 | 33 | 3. To run the application server: 34 | ```bash 35 | streamlit run app.py 36 | ``` 37 | 38 | 4. If you want to generate personalized cold outreach openers for multiple people from a CSV file. 39 | 40 | ```bash 41 | python main_batch.py 42 | # Or with custom files 43 | python main_batch.py --input my_targets.csv --output my_results.csv 44 | ``` 45 | 46 | CSV file with three required columns: 47 | - `first_name`: Target person's first name 48 | - `last_name`: Target person's last name 49 | - `keywords`: Space-separated keywords (e.g., "Tesla SpaceX entrepreneur") 50 | 51 | CSV file with original columns plus: 52 | - `opening_message`: Generated personalized message 53 | - `search_results`: Comma-separated list of URLs 54 | - For each personalization rule (e.g., personal_connection, recent_promotion, recent_talks): 55 | - `rule_actionable`: Whether the rule could be applied (True/False) 56 | - `rule_details`: Supporting details if rule was actionable 57 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from flow import cold_outreach_flow 2 | 3 | def main(): 4 | """ 5 | Main function to run the Cold Outreach Opener Generator application. 6 | """ 7 | # Sample input data 8 | shared = { 9 | "input": { 10 | "first_name": "Elon", 11 | "last_name": "Musk", 12 | "keywords": "Tesla", 13 | "personalization_factors": [ 14 | { 15 | "name": "personal_connection", 16 | "description": "Check if the target person is from the University of Pennsylvania", 17 | "action": "If they are, say 'Go Quakers!'" 18 | }, 19 | { 20 | "name": "recent_achievement", 21 | "description": "Check if the target person was recently promoted", 22 | "action": "If they were, say 'Congratulations on your recent promotion...'" 23 | }, 24 | { 25 | "name": "shared_interest", 26 | "description": "Check if the target person is interested in sustainable energy", 27 | "action": "If they are, say 'I've been following your work on sustainable energy...'" 28 | }, 29 | { 30 | "name": "recent_talks", 31 | "description": "Check if the target person gave a recent talk", 32 | "action": "If they did, say 'I heard you gave a recent talk on...'" 33 | } 34 | ], 35 | "style": """Be concise, specific, and casual in 30 words or less. For example: 36 | 'Heard about your talk on the future of space exploration—loved your take on creating a more sustainable path for space travel.'""" 37 | } 38 | } 39 | 40 | print("Generating personalized cold outreach opener...") 41 | print(f"Target person: {shared['input']['first_name']} {shared['input']['last_name']}") 42 | print(f"Keywords: {shared['input']['keywords']}") 43 | print(f"Style: {shared['input']['style']}") 44 | print("\nSearching the web for information...") 45 | 46 | # Run the flow 47 | cold_outreach_flow.run(shared) 48 | 49 | # Display the results 50 | print("\n--- PERSONALIZATION INSIGHTS ---") 51 | if shared.get("personalization"): 52 | for factor, details in shared["personalization"].items(): 53 | print(f"\n• {factor.upper()}:") 54 | print(f" Details: {details['details']}") 55 | print(f" Action: {details['action']}") 56 | else: 57 | print("No personalization factors were actionable.") 58 | 59 | print("\n--- GENERATED OPENING MESSAGE ---") 60 | print(shared["output"]["opening_message"]) 61 | 62 | if __name__ == "__main__": 63 | main() -------------------------------------------------------------------------------- /docs/design.md: -------------------------------------------------------------------------------- 1 | # Cold Outreach Opener Generator - Design Document 2 | 3 | ## 1. Project Requirements 4 | 5 | The Cold Outreach Opener Generator creates personalized opening messages for cold outreach emails based on: 6 | 7 | - **Target Person Information**: First name, last name, and space-separated keywords related to the person (e.g., "Tesla SpaceX entrepreneur") 8 | - **Personalization Factors**: List of things to look for (e.g., personal connections, recent promotions, talks)Example factors: 9 | 1. **Personal connection**: "If target person has Columbia University affiliation, mention our shared connection to Columbia" 10 | 2. **Recent promotion**: "If target person was recently promoted, congratulate them on their new role" 11 | 3. **Recent talks or publications**: "If target person gave talks recently, mention enjoying their insights" 12 | - **Tone Preferences**: Desired message style (e.g., concise, friendly, casual) 13 | 14 | ## 2. Utility Functions 15 | 16 | Following the "start small" principle, we've implemented these essential utility functions: 17 | 18 | - `call_llm(prompt)` in `utils/call_llm.py` 19 | 20 | - `search_web(query)` in `utils/search_web.py` 21 | - **Purpose**: General web search function to find information 22 | - **Input**: Search query string 23 | - **Output**: List of search results (snippets and URLs) 24 | - **Implementation**: Uses Google Custom Search API to perform searches 25 | 26 | - `get_html_content(url)` in `utils/content_retrieval.py` 27 | - **Purpose**: Retrieve HTML content from a URL 28 | - **Input**: URL string 29 | - **Output**: HTML content or extracted text from the webpage 30 | - **Implementation**: Uses requests library to fetch content from the URL 31 | 32 | ## 3. Flow Architecture 33 | 34 | Based on our utility functions, the flow will consist of these nodes: 35 | 36 | ### SearchPersonNode 37 | - **Purpose**: Search for information about the target person 38 | - **Design**: Regular Node 39 | - **Data Access**: 40 | - **Prep**: Read first_name, last_name, keywords from shared store 41 | - **Exec**: Call search_web utility with a formatted query like "FirstName LastName Keywords" to retrieve top 10 URLs (note: keywords is a space-separated string, not a list) 42 | - **Post**: Write search results to shared store 43 | 44 | ### ContentRetrievalNode 45 | - **Purpose**: Retrieve content from each search result URL 46 | - **Design**: BatchNode (processes each URL separately) 47 | - **Data Access**: 48 | - **Prep**: Read search results from shared store and return list of URLs 49 | - **Exec**: For each URL, call get_html_content; if retrieval fails, return empty content 50 | - **Post**: Write only non-empty webpage contents to shared store, filtering out failed retrievals 51 | 52 | ### AnalyzeResultsBatchNode 53 | - **Purpose**: Analyze each webpage content for personalization factors 54 | - **Design**: BatchNode (processes each URL content separately) 55 | - **Data Access**: 56 | - **Prep**: Return list of (url, content) pairs from shared["webpage_contents"] 57 | - **Exec**: For each content, call LLM to analyze and extract relevant personalization details 58 | - **Post**: Combine all actionable personalization factors and write to shared store 59 | 60 | ### DraftOpeningNode 61 | - **Purpose**: Generate personalized opening message 62 | - **Design**: Regular Node 63 | - **Data Access**: 64 | - **Prep**: Read target person info, actionable personalization factors, and style preferences 65 | - **Exec**: Call LLM to draft opening message based on the specified style 66 | - **Post**: Write draft opening message to shared store 67 | 68 | ## Flow Sequence 69 | 70 | 71 | ```mermaid 72 | flowchart LR 73 | A[SearchPersonNode] --> B[ContentRetrievalNode] 74 | B --> C[AnalyzeResultsBatchNode] 75 | C --> D[DraftOpeningNode] 76 | 77 | classDef batch fill:#f9f,stroke:#333,stroke-width:2px 78 | class B,C batch 79 | ``` 80 | 81 | The diagram visually represents our flow, with batch nodes highlighted. 82 | 83 | ## 4. Data Schema 84 | 85 | The shared store will contain: 86 | 87 | ```python 88 | shared = { 89 | "input": { 90 | "first_name": str, # Target person's first name 91 | "last_name": str, # Target person's last name 92 | "keywords": str, # Space-separated keywords related to the person 93 | "personalization_factors": list[dict], # Things to watch for and corresponding actions 94 | "style": str # Desired message style 95 | }, 96 | "search_results": list[dict], # Results from web search 97 | "webpage_contents": dict, # HTML/text content of relevant pages, keyed by URL 98 | "personalization": { 99 | # Each key corresponds to a found personalization factor 100 | "factor_name": { 101 | "actionable": bool, # Whether the factor was actionable 102 | "details": str, # Supporting details if actionable 103 | "action": str # Corresponding action to take 104 | } 105 | }, 106 | "output": { 107 | "opening_message": str # The generated opening message 108 | } 109 | } 110 | ``` 111 | -------------------------------------------------------------------------------- /output.csv: -------------------------------------------------------------------------------- 1 | first_name,last_name,keywords,opening_message,search_results,recent_promotion_actionable,recent_promotion_details,recent_talks_actionable,recent_talks_details 2 | Elon,Musk,Tesla SpaceX entrepreneur,"Elon, congratulations on heading DOGE! Caught your recent Rogan interview—your insights on streamlining government efficiency while balancing innovation priorities were fascinating.","https://www.tesla.com/elon-musk,https://en.wikipedia.org/wiki/Elon_Musk,https://www.investopedia.com/articles/personal-finance/061015/how-elon-musk-became-elon-musk.asp,https://x.com/elonmusk?lang=en,https://www.biography.com/business-leaders/elon-musk,https://www.reddit.com/r/IAmA/comments/kkn5v/ask_tesla_spacex_and_paypal_cofounder_elon_musk/,https://twitter.com/elonmusk,https://www.reddit.com/r/SpaceXMasterrace/comments/sasbcw/elon_musk_was_not_the_founder_of_spacex_he_took/,https://www.space.com/18849-elon-musk.html,https://www.forbes.com/profile/elon-musk/",True,"Elon Musk was appointed as a senior advisor to United States president Donald Trump and de facto head of the Department of Government Efficiency (DOGE) in January 2025. | The content mentions Elon Musk's very recent appointment as (unofficial) head of the so-called Department of Government Efficiency (DOGE) following the 2024 election, where he now has 'unprecedented sway over federal policy and spending.' This appears to be a significant new role for him in government. | Elon Musk was recently appointed to lead the Department of Government Efficiency (DOGE) under President Trump's second administration in January 2025. This is a temporary organization tasked with cutting unnecessary federal spending and modernizing technology within the federal government. | Elon Musk recently became part of the government as head of the Department of Government Efficiency (DOGE). The content references his 'new government role' and discusses his activities leading DOGE, including sending emails to federal employees.",True,"Musk recently spoke at Trump's Presidential Parade in January 2025, where he made controversial remarks and gestures. He also gave an interview with psychologist Jordan Peterson in 2024 where he discussed his religious views, stating he considers himself a 'cultural Christian' and believes in the teachings of Jesus Christ. | Musk was recently interviewed by Joe Rogan (mentioned as 'Joe Rogan Interviews Elon Musk: Here Are The Key Moments' from February 28, 2025) and also spoke at the Conservative Political Action Conference on February 20, 2025." 3 | Sundar,Pichai,Google CEO tech,"Hi Sundar, 4 | 5 | Your recent lecture at Carnegie Mellon about AI becoming a ""natural extension"" of daily life really resonated with me, especially your insights on Project Astra and DeepMind's mathematical breakthroughs.","https://en.wikipedia.org/wiki/Sundar_Pichai,https://www.linkedin.com/in/sundarpichai,https://mitsloan.mit.edu/ideas-made-to-matter/googles-sundar-pichai-says-tech-a-powerful-agent-change,https://www.productplan.com/learn/sundar-pichai-product-manager-to-ceo/,https://blog.google/authors/sundar-pichai/,https://www.weforum.org/stories/2020/01/this-is-how-quantum-computing-will-change-our-lives-8a0d33657f/,https://www.cmu.edu/news/stories/archives/2024/september/google-ceo-sundar-pichai-launches-the-2024-25-presidents-lecture-series-at-carnegie-mellon,https://www.nytimes.com/2018/11/08/business/sundar-pichai-google-corner-office.html,https://www.youtube.com/watch?v=OsxwBmp3iFU,https://www.theguardian.com/technology/2015/aug/10/google-sundar-pichai-ceo-alphabet",True,"The article announces Sundar Pichai's promotion to CEO of Google as part of a restructuring where Google became a subsidiary of Alphabet, with Larry Page moving to head Alphabet. This was announced on August 10, 2015.",True,"The content describes Pichai giving a talk at MIT Forefront where he discussed AI, future of work, diversity and inclusion, and climate change with MIT president L. Rafael Reif in March 2022 | The content mentions 'Google I/O 2024' where Sundar Pichai shared news about updates across Gemini, Android, Search and Photos, which indicates he recently gave a presentation or talk at this event. | Sundar Pichai recently gave a lecture at Carnegie Mellon University on September 18, 2024, titled 'The AI Platform Shift and the Opportunity Ahead' as part of the President's Lecture Series. He discussed AI innovations including Project Astra, Google DeepMind's success in the International Mathematical Olympiad, and his vision for AI becoming a 'natural extension' in everyday life. | The webpage is about a talk/interview titled 'Building the Future: Sundar Pichai on A.I., Regulation and What's Next for Google' which appears to be a recent discussion where he shared insights about AI, regulation, and Google's future." 6 | Tim,Cook,Apple iPhone CEO ,"Tim, your recent London interview about your Apple journey was refreshing. I especially appreciated your candid reflections on how Steve Jobs recruited you—it offers valuable perspective for today's tech leaders.","https://en.wikipedia.org/wiki/Tim_Cook,https://www.apple.com/leadership/tim-cook/,https://www.youtube.com/watch?v=m4RVTK7iU1c,https://www.apple.com/leadership/,https://x.com/tim_cook/status/1890068457825394918?lang=en,https://stratechery.com/2013/tim-cook-is-a-great-ceo/,https://x.com/tim_cook?lang=en,https://www.reddit.com/r/apple/comments/xa0vg9/tim_cook_has_been_ceo_of_apple_since_2011_since/,https://investor.apple.com/our_values/default.aspx,https://www.forbes.com/profile/tim-cook/",,,True,"The webpage shows Tim Cook giving an interview titled 'The Job Interview' where he discusses how Steve Jobs recruited him and other topics. This indicates a recent talk/interview that could be referenced. | Tim Cook was interviewed in London recently (mentioned in Dec 14, 2024 article: 'Apple CEO Tim Cook Declares New iPhone Era In Detailed Interview'). The interview covered business advice, his personal story, and Apple Intelligence." 7 | -------------------------------------------------------------------------------- /main_batch.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import argparse 3 | import os 4 | import json 5 | from flow import cold_outreach_flow 6 | 7 | def main(): 8 | """ 9 | Batch processing script for the Cold Outreach Opener Generator. 10 | Processes multiple people from a CSV file and generates personalized opening messages. 11 | """ 12 | # Parse command line arguments 13 | parser = argparse.ArgumentParser(description='Process multiple cold outreach targets from a CSV file.') 14 | parser.add_argument('--input', default='input.csv', help='Input CSV file (default: input.csv)') 15 | parser.add_argument('--output', default='output.csv', help='Output CSV file (default: output.csv)') 16 | args = parser.parse_args() 17 | 18 | # Check if input file exists 19 | if not os.path.exists(args.input): 20 | print(f"Error: Input file '{args.input}' not found.") 21 | return 22 | 23 | # Hardcoded personalization factors 24 | personalization_factors = [ 25 | { 26 | "name": "personal_connection", 27 | "description": "Check if target person has Columbia University affiliation", 28 | "action": "If they do, mention shared connection to Columbia" 29 | }, 30 | { 31 | "name": "recent_promotion", 32 | "description": "Check if target person was recently promoted", 33 | "action": "If they were, congratulate them on their new role" 34 | }, 35 | { 36 | "name": "recent_talks", 37 | "description": "Check if target person gave talks recently", 38 | "action": "If they did, mention enjoying their insights" 39 | } 40 | ] 41 | 42 | # Extract factor names for later use 43 | factor_names = [factor["name"] for factor in personalization_factors] 44 | print(factor_names) 45 | 46 | # Hardcoded style preference 47 | style = "Be concise, specific, and casual in 30 words or less. For example: 'Heard about your talk on the future of space exploration—loved your take on creating a more sustainable path for space travel.'" 48 | 49 | # Read input CSV 50 | input_data = [] 51 | with open(args.input, 'r', newline='', encoding='utf-8') as csvfile: 52 | reader = csv.DictReader(csvfile) 53 | for row in reader: 54 | if 'first_name' in row and 'last_name' in row and 'keywords' in row: 55 | input_data.append(row) 56 | 57 | if not input_data: 58 | print(f"Error: No valid data found in '{args.input}'. CSV should have columns: first_name, last_name, keywords") 59 | return 60 | 61 | # Process each person 62 | results = [] 63 | total = len(input_data) 64 | for i, person in enumerate(input_data, 1): 65 | print(f"\nProcessing {i}/{total}: {person['first_name']} {person['last_name']}") 66 | 67 | # Prepare input data 68 | shared = { 69 | "input": { 70 | "first_name": person['first_name'], 71 | "last_name": person['last_name'], 72 | "keywords": person['keywords'], 73 | "personalization_factors": personalization_factors, 74 | "style": style 75 | } 76 | } 77 | 78 | # Run the flow 79 | try: 80 | cold_outreach_flow.run(shared) 81 | 82 | # Prepare result row 83 | # Extract URLs as comma-separated string 84 | urls = [result.get("link", "") for result in shared.get("search_results", []) if "link" in result] 85 | url_string = ",".join(urls) 86 | 87 | # Extract personalization details 88 | personalization_data = {} 89 | for factor_name, details in shared.get("personalization", {}).items(): 90 | personalization_data[factor_name + "_actionable"] = str(details.get("actionable", False)) 91 | personalization_data[factor_name + "_details"] = details.get("details", "") 92 | 93 | result = { 94 | 'first_name': person['first_name'], 95 | 'last_name': person['last_name'], 96 | 'keywords': person['keywords'], 97 | 'opening_message': shared.get("output", {}).get("opening_message", ""), 98 | 'search_results': url_string, 99 | **personalization_data # Add all personalization fields 100 | } 101 | results.append(result) 102 | 103 | # Display the result 104 | print(f"Generated opener: {result['opening_message']}") 105 | 106 | except Exception as e: 107 | print(f"Error processing {person['first_name']} {person['last_name']}: {str(e)}") 108 | # Add failed row with error message 109 | results.append({ 110 | 'first_name': person['first_name'], 111 | 'last_name': person['last_name'], 112 | 'keywords': person['keywords'], 113 | 'opening_message': f"ERROR: {str(e)}", 114 | 'search_results': "", 115 | # Include empty personalization fields for consistency with successful rows 116 | **{f"{factor}_actionable": "False" for factor in factor_names}, 117 | **{f"{factor}_details": "" for factor in factor_names} 118 | }) 119 | 120 | # Write results to output CSV 121 | if results: 122 | # Determine all field names by examining all the result rows 123 | all_fields = set() 124 | for result in results: 125 | all_fields.update(result.keys()) 126 | 127 | fieldnames = ['first_name', 'last_name', 'keywords', 'opening_message', 'search_results'] 128 | # Add personalization fields in a specific order 129 | for factor in factor_names: 130 | if f"{factor}_actionable" in all_fields: 131 | fieldnames.append(f"{factor}_actionable") 132 | if f"{factor}_details" in all_fields: 133 | fieldnames.append(f"{factor}_details") 134 | 135 | with open(args.output, 'w', newline='', encoding='utf-8') as csvfile: 136 | writer = csv.DictWriter(csvfile, fieldnames=fieldnames) 137 | writer.writeheader() 138 | writer.writerows(results) 139 | 140 | print(f"\nProcessing complete. Results written to '{args.output}'") 141 | else: 142 | print("\nNo results to write.") 143 | 144 | if __name__ == "__main__": 145 | main() -------------------------------------------------------------------------------- /flow.py: -------------------------------------------------------------------------------- 1 | from pocketflow import Node, BatchNode, Flow 2 | from utils.call_llm import call_llm 3 | from utils.search_web import search_web 4 | from utils.content_retrieval import get_html_content 5 | import logging 6 | import sys 7 | 8 | # Configure logging 9 | logging.basicConfig( 10 | level=logging.INFO, 11 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', 12 | handlers=[ 13 | logging.StreamHandler(sys.stdout) 14 | ] 15 | ) 16 | logging.getLogger("httpx").setLevel(logging.WARNING) 17 | logger = logging.getLogger("personalization_flow") 18 | 19 | 20 | class SearchPersonNode(Node): 21 | def prep(self, shared): 22 | # Read target person info from shared store 23 | first_name = shared["input"]["first_name"] 24 | last_name = shared["input"]["last_name"] 25 | keywords = shared["input"]["keywords"] 26 | 27 | # Format the search query 28 | query = f"{first_name} {last_name} {keywords}" 29 | logger.info(f"Prepared search query: '{query}'") 30 | return query 31 | 32 | def exec(self, query): 33 | # Execute web search 34 | logger.info(f"Executing web search with query: '{query}'") 35 | search_results = search_web(query) 36 | logger.debug(f"Search returned {len(search_results)} results") 37 | return search_results 38 | 39 | def post(self, shared, prep_res, exec_res): 40 | # Store search results in shared store 41 | shared["search_results"] = exec_res 42 | logger.info(f"Stored {len(exec_res)} search results in shared store") 43 | return "default" 44 | 45 | 46 | class ContentRetrievalNode(BatchNode): 47 | def prep(self, shared): 48 | # Get list of URLs from search results 49 | search_results = shared["search_results"] 50 | urls = [result["link"] for result in search_results if "link" in result] 51 | logger.info(f"Preparing to retrieve content from {len(urls)} URLs") 52 | return urls 53 | 54 | def exec(self, url): 55 | # Retrieve content from URL 56 | logger.debug(f"Retrieving content from URL: {url}") 57 | content = get_html_content(url) 58 | return {"url": url, "content": content} 59 | 60 | def exec_fallback(self, prep_res, exc): 61 | # This is called after all retries are exhausted 62 | url = prep_res["url"] # Extract URL from the prep_res input pair 63 | logger.error(f"Failed to retrieve content from {url} after all retries: {exc}") 64 | return {"url": url, "content": None} 65 | 66 | def post(self, shared, prep_res, exec_res_list): 67 | # Store only non-empty webpage contents 68 | valid_contents = [res for res in exec_res_list if res["content"]] 69 | shared["web_contents"] = valid_contents 70 | logger.info(f"Retrieved content from {len(valid_contents)}/{len(exec_res_list)} URLs successfully") 71 | return "default" 72 | 73 | 74 | class AnalyzeResultsBatchNode(BatchNode): 75 | def prep(self, shared): 76 | # Store first_name, last_name, and personalization_factors for exec 77 | self.first_name = shared["input"]["first_name"] 78 | self.last_name = shared["input"]["last_name"] 79 | self.personalization_factors = shared["input"]["personalization_factors"] 80 | 81 | # Return list of (url, content) pairs 82 | url_content_pairs = list(shared["web_contents"]) 83 | logger.info(f"Analyzing content from {len(url_content_pairs)} web pages") 84 | return url_content_pairs 85 | 86 | def exec(self, url_content_pair): 87 | url, content = url_content_pair["url"], url_content_pair["content"] 88 | logger.debug(f"Analyzing content from: {url}") 89 | 90 | # Prepare prompt for LLM analysis 91 | prompt = f"""Analyze the following webpage content about {self.first_name} {self.last_name}. 92 | Look for the following personalization factors: 93 | {self._format_personalization_factors(self.personalization_factors)} 94 | 95 | Content from {url}: 96 | Title: {content["title"]} 97 | 98 | Text: 99 | {content["text"]} # Limit text to avoid overly large prompts 100 | 101 | For each factor, return if you found relevant information and details. 102 | Format your response as YAML: 103 | ```yaml 104 | factors: 105 | - name: "factor_name" 106 | action: "action to take" 107 | actionable: true/false 108 | details: "supporting details if actionable" 109 | - name: "another_factor" 110 | action: "action to take" 111 | actionable: true/false 112 | details: "supporting details if actionable" 113 | ```""" 114 | 115 | # Call LLM to analyze the content 116 | logger.debug(f"Calling LLM to analyze content from {url}") 117 | response = call_llm(prompt) 118 | 119 | # Extract YAML portion from the response 120 | import yaml 121 | yaml_part = response.split("```yaml")[1].split("```")[0].strip() 122 | analysis = yaml.safe_load(yaml_part) 123 | logger.debug(f"Successfully parsed YAML from LLM response for {url}") 124 | return {"url": url, "analysis": analysis} 125 | 126 | def exec_fallback(self, prep_res, exc): 127 | # This is called after all retries are exhausted 128 | url = prep_res["url"] # Extract URL from the prep_res input pair 129 | logger.error(f"Failed to analyze content from {url} after all retries: {exc}") 130 | return {"url": url, "analysis": {"factors": []}} 131 | 132 | def _format_personalization_factors(self, factors): 133 | formatted = "" 134 | for i, factor in enumerate(factors): 135 | formatted += f"{i+1}. {factor['name']}: {factor['description']}\n Action: {factor['action']}\n" 136 | return formatted 137 | 138 | def post(self, shared, prep_res, exec_res_list): 139 | # Initialize personalization in shared store 140 | shared["personalization"] = {} 141 | 142 | # Process all analysis results 143 | found_factors = 0 144 | total_factors = 0 145 | 146 | # Dictionary to temporarily store details for each factor across sources 147 | factor_details = {} 148 | 149 | for result in exec_res_list: 150 | if "analysis" in result and "factors" in result["analysis"]: 151 | for factor in result["analysis"]["factors"]: 152 | total_factors += 1 153 | if factor.get("actionable", False): 154 | found_factors += 1 155 | factor_name = factor["name"] 156 | 157 | # Initialize if first time seeing this factor 158 | if factor_name not in factor_details: 159 | factor_details[factor_name] = [] 160 | 161 | # Add details from this source 162 | factor_details[factor_name].append(factor["details"]) 163 | 164 | # Process collected details and create final personalization entries 165 | for factor_name, details_list in factor_details.items(): 166 | # Find the matching factor from input to get the action 167 | for input_factor in shared["input"]["personalization_factors"]: 168 | if input_factor["name"] == factor_name: 169 | # Merge all details for this factor 170 | merged_details = " | ".join(details_list) 171 | 172 | shared["personalization"][factor_name] = { 173 | "actionable": True, 174 | "details": merged_details, 175 | "action": input_factor["action"] 176 | } 177 | logger.debug(f"Found information for factor: {factor_name}") 178 | break 179 | 180 | logger.info(f"Analysis complete: Found information for {found_factors}/{total_factors} factors across {len(exec_res_list)} sources") 181 | return "default" 182 | 183 | 184 | class DraftOpeningNode(Node): 185 | def prep(self, shared): 186 | # Gather all necessary information 187 | person_info = { 188 | "first_name": shared["input"]["first_name"], 189 | "last_name": shared["input"]["last_name"] 190 | } 191 | 192 | personalization = shared["personalization"] 193 | style = shared["input"]["style"] 194 | 195 | logger.info(f"Preparing to draft opening message for {person_info['first_name']} {person_info['last_name']}") 196 | logger.debug(f"Found {len(personalization)} personalization factors to include") 197 | return person_info, personalization, style 198 | 199 | def exec(self, prep_data): 200 | person_info, personalization, style = prep_data 201 | 202 | # Prepare prompt for LLM 203 | prompt = f"""Generate a personalized opening message for a cold outreach email to {person_info["first_name"]} {person_info["last_name"]}. 204 | 205 | Based on our research, we found the following personalization factors: 206 | {self._format_personalization_details(personalization)} 207 | 208 | Style preferences: {style} 209 | 210 | Write a concise opening paragraph (1-3 sentences) that: 211 | 1. Addresses the person by first name 212 | 2. Includes the personalization points we found 213 | 3. Matches the requested style 214 | 4. Feels authentic and not forced 215 | 216 | Only return the opening message, nothing else.""" 217 | 218 | # Call LLM to draft the opening 219 | logger.debug("Calling LLM to draft personalized opening message") 220 | return call_llm(prompt) 221 | 222 | def _format_personalization_details(self, personalization): 223 | if not personalization: 224 | return "No specific personalization factors were actionable." 225 | 226 | formatted = "" 227 | for factor_name, details in personalization.items(): 228 | formatted += f"- {factor_name}: {details['details']}\n Action: {details['action']}\n" 229 | return formatted 230 | 231 | def post(self, shared, prep_res, exec_res): 232 | # Store the opening message in the output 233 | if "output" not in shared: 234 | shared["output"] = {} 235 | shared["output"]["opening_message"] = exec_res 236 | logger.info("Successfully generated and stored personalized opening message") 237 | return "default" 238 | 239 | 240 | # Create nodes 241 | logger.info("Initializing flow nodes") 242 | search_node = SearchPersonNode() 243 | content_node = ContentRetrievalNode() 244 | analyze_node = AnalyzeResultsBatchNode(max_retries=2, wait=10) # Retry up to 3 times before using fallback 245 | draft_node = DraftOpeningNode(max_retries=3, wait=10) 246 | 247 | # Connect nodes in the flow 248 | logger.info("Connecting nodes in personalization flow") 249 | search_node >> content_node >> analyze_node >> draft_node 250 | 251 | # Create the flow 252 | cold_outreach_flow = Flow(start=search_node) 253 | logger.info("Personalization flow initialized successfully") -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import logging 3 | import sys 4 | from flow import cold_outreach_flow 5 | 6 | # Configure logging 7 | logging.basicConfig( 8 | level=logging.INFO, 9 | format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', 10 | handlers=[ 11 | logging.StreamHandler(sys.stdout) 12 | ] 13 | ) 14 | logger = logging.getLogger("streamlit_app") 15 | 16 | # Function to validate minimum length 17 | def validate_min_length(text, min_length, field_name): 18 | if text and len(text) < min_length: 19 | return f"{field_name} must be at least {min_length} characters." 20 | return None 21 | 22 | def main(): 23 | st.set_page_config( 24 | page_title="Cold Outreach Opener Generator", 25 | page_icon="✉️", 26 | layout="wide" 27 | ) 28 | 29 | st.title("Cold Outreach Opener Generator") 30 | st.write("Generate personalized opening messages for cold outreach emails based on web search results.") 31 | 32 | # Sidebar for detailed app description 33 | with st.sidebar: 34 | st.subheader("About") 35 | st.write(""" 36 | This app automatically generates personalized opening messages for cold outreach emails 37 | based on web search results. It uses LLMs and web search to find relevant information 38 | about the target person. 39 | """) 40 | st.write("---") 41 | st.subheader("How it works:") 42 | st.write(""" 43 | 1. Enter details about the target person 44 | 2. Define personalization factors to look for 45 | 3. Set your preferred message style 46 | 4. Click "Generate Opening" to search the web and generate a personalized message 47 | """) 48 | 49 | st.write("---") 50 | st.subheader("Open Source") 51 | st.write(""" 52 | This project is fully open sourced at: [GitHub](https://github.com/The-Pocket/Tutorial-Cold-Email-Personalization) 53 | 54 | This is an example LLM project for [Pocket Flow](https://github.com/The-Pocket/PocketFlow), a 100-line LLM framework 55 | """) 56 | 57 | # Main interface - inputs 58 | col1, col2 = st.columns([1, 1]) 59 | 60 | # Error message containers 61 | error_container = st.empty() 62 | errors = [] 63 | 64 | with col1: 65 | st.subheader("Target Person Information") 66 | 67 | # Create two columns for first name and last name 68 | name_col1, name_col2 = st.columns(2) 69 | 70 | with name_col1: 71 | first_name = st.text_input("First Name (1-30 chars)", "Elon", max_chars=30) 72 | if err := validate_min_length(first_name, 1, "First name"): 73 | errors.append(err) 74 | 75 | with name_col2: 76 | last_name = st.text_input("Last Name (1-30 chars)", "Musk", max_chars=30) 77 | if err := validate_min_length(last_name, 1, "Last name"): 78 | errors.append(err) 79 | 80 | keywords = st.text_input("Keywords (max 100 chars)", "Tesla", max_chars=100) 81 | 82 | st.subheader("Message Style") 83 | style = st.text_area("Style Preferences (10-500 chars)", """Be concise, specific, and casual in 30 words or less. For example: 'Heard about your talk on the future of space exploration—loved your take on creating a more sustainable path for space travel.'""", height=150, max_chars=500) 84 | if err := validate_min_length(style, 10, "Style preferences"): 85 | errors.append(err) 86 | 87 | with col2: 88 | st.subheader("Personalization Factors") 89 | st.write("Define what personal information to look for and how to use it in your message (1-5 factors allowed).") 90 | 91 | # Initialize session state for personalization factors if not exists 92 | if 'personalization_factors' not in st.session_state: 93 | st.session_state.personalization_factors = [ 94 | { 95 | "name": "personal_connection", 96 | "description": "Check if the target person is from the University of Pennsylvania", 97 | "action": "If they are, say 'Go Quakers!'" 98 | }, 99 | { 100 | "name": "recent_achievement", 101 | "description": "Check if the target person was recently promoted", 102 | "action": "Say 'Congratulations on your recent promotion...'" 103 | }, 104 | ] 105 | 106 | # Display existing factors 107 | for i, factor in enumerate(st.session_state.personalization_factors): 108 | with st.expander(f"Factor {i+1}: {factor['name']}", expanded=False): 109 | factor_name = st.text_input("Name (5-30 chars)", factor["name"], key=f"name_{i}", max_chars=30) 110 | if err := validate_min_length(factor_name, 5, f"Factor {i+1} name"): 111 | errors.append(err) 112 | 113 | factor_desc = st.text_input("Description (10-100 chars)", factor["description"], key=f"desc_{i}", max_chars=100) 114 | if err := validate_min_length(factor_desc, 10, f"Factor {i+1} description"): 115 | errors.append(err) 116 | 117 | factor_action = st.text_input("Action (10-100 chars)", factor["action"], key=f"action_{i}", max_chars=100) 118 | if err := validate_min_length(factor_action, 10, f"Factor {i+1} action"): 119 | errors.append(err) 120 | 121 | # Create two columns for buttons 122 | col1, col2 = st.columns(2) 123 | 124 | with col1: 125 | if st.button("Update", key=f"update_{i}", use_container_width=True): 126 | # Validate minimum lengths before updating 127 | if (len(factor_name) >= 5 and len(factor_desc) >= 10 and len(factor_action) >= 10): 128 | st.session_state.personalization_factors[i] = { 129 | "name": factor_name, 130 | "description": factor_desc, 131 | "action": factor_action 132 | } 133 | st.success("Factor updated!") 134 | else: 135 | st.error("Please fix validation errors before updating.") 136 | 137 | with col2: 138 | if st.button("Remove", key=f"remove_{i}", type="primary", use_container_width=True): 139 | st.session_state.personalization_factors.pop(i) 140 | st.rerun() 141 | 142 | # Add new factor (if we're under the maximum of 5) 143 | if len(st.session_state.personalization_factors) < 5: 144 | with st.expander("Add New Factor", expanded=False): 145 | new_name = st.text_input("Name (5-30 chars)", "shared_interest", key="new_name", max_chars=30) 146 | if err := validate_min_length(new_name, 5, "New factor name"): 147 | errors.append(err) 148 | 149 | new_desc = st.text_input("Description (10-100 chars)", "Check if the target person is interested in...", key="new_desc", max_chars=100) 150 | if err := validate_min_length(new_desc, 10, "New factor description"): 151 | errors.append(err) 152 | 153 | new_action = st.text_input("Action (10-100 chars)", "Say 'I've been following your work on...'", key="new_action", max_chars=100) 154 | if err := validate_min_length(new_action, 10, "New factor action"): 155 | errors.append(err) 156 | 157 | if st.button("Add Factor", type="primary", use_container_width=True): 158 | # Validate minimum lengths before adding 159 | if (len(new_name) >= 5 and len(new_desc) >= 10 and len(new_action) >= 10): 160 | st.session_state.personalization_factors.append({ 161 | "name": new_name, 162 | "description": new_desc, 163 | "action": new_action 164 | }) 165 | st.success("Factor added!") 166 | st.rerun() 167 | else: 168 | st.error("Please fix validation errors before adding.") 169 | else: 170 | st.warning("Maximum of 5 personalization factors reached. Remove one to add a new factor.") 171 | 172 | # Display validation errors if any 173 | if errors: 174 | error_container.error("\n".join(errors)) 175 | 176 | # Check factor count 177 | if len(st.session_state.personalization_factors) < 1: 178 | error_container.error("At least 1 personalization factor is required.") 179 | generate_disabled = True 180 | else: 181 | generate_disabled = bool(errors) 182 | 183 | # Generate button 184 | if st.button("Generate Opening", type="primary", use_container_width=True, disabled=generate_disabled): 185 | if not first_name or not last_name: 186 | st.error("Please provide at least the person's first and last name.") 187 | return 188 | 189 | # Create the shared data structure 190 | shared = { 191 | "input": { 192 | "first_name": first_name, 193 | "last_name": last_name, 194 | "keywords": keywords, 195 | "personalization_factors": st.session_state.personalization_factors, 196 | "style": style 197 | } 198 | } 199 | 200 | # Show progress 201 | with st.spinner("Searching the web for information about the target person..."): 202 | # Create a status area 203 | status_area = st.empty() 204 | log_area = st.empty() 205 | 206 | # Create a custom log handler to display logs in Streamlit 207 | log_messages = [] 208 | class StreamlitLogHandler(logging.Handler): 209 | def emit(self, record): 210 | log_messages.append(self.format(record)) 211 | log_text = "\n".join(log_messages[-10:]) # Show last 10 messages 212 | log_area.code(log_text, language="bash") 213 | 214 | # Add the custom handler to the logger 215 | streamlit_handler = StreamlitLogHandler() 216 | streamlit_handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) 217 | logging.getLogger().addHandler(streamlit_handler) 218 | 219 | try: 220 | # Run the flow 221 | cold_outreach_flow.run(shared) 222 | 223 | # Remove the custom handler 224 | logging.getLogger().removeHandler(streamlit_handler) 225 | 226 | except Exception as e: 227 | status_area.error(f"An error occurred: {str(e)}") 228 | logging.getLogger().removeHandler(streamlit_handler) 229 | st.error(f"Failed to generate opening message: {str(e)}") 230 | return 231 | 232 | # Display results 233 | if "output" in shared and "opening_message" in shared["output"]: 234 | st.success(shared["output"]["opening_message"]) 235 | 236 | # Display personalization details 237 | if "personalization" in shared and shared["personalization"]: 238 | st.subheader("Personalization Details Found") 239 | for factor_name, details in shared["personalization"].items(): 240 | with st.expander(f"Factor: {factor_name}"): 241 | st.write(f"**Details:** {details['details']}") 242 | st.write(f"**Action:** {details['action']}") 243 | else: 244 | st.info("No personalization factors were found for this person.") 245 | else: 246 | st.warning("No opening message was generated. Check the logs for details.") 247 | 248 | if __name__ == "__main__": 249 | main() -------------------------------------------------------------------------------- /blog.md: -------------------------------------------------------------------------------- 1 | # Use AI to Generate Cold Outreach Openers: Step-by-Step Tutorial 2 | 3 | ![Header image showing personalized cold email generation process](./assets/banner.png) 4 | 5 | Cold outreach is a numbers game—but that doesn't mean it has to feel like spam. 6 | 7 | What if you could personally research **each prospect**, find their recent achievements, interests, and background, and craft a thoughtful opening message that shows you've done your homework? 8 | 9 | That's exactly what we're building today: a tool that uses AI to automate what would normally take hours of manual research and writing. In this tutorial, I'll show you how to use AI to generate cold outreach openers that are: 10 | 11 | - **Actually personalized** (not just "Hey {first_name}!") 12 | - **Based on real research** (not made-up facts) 13 | - **Attention-grabbing** (by referencing things your prospect actually cares about) 14 | 15 | The best part? You can adapt this approach for your own needs—whether you're looking for a job, raising funds for your startup, or reaching out to potential clients. 16 | 17 | Let's dive in. 18 | 19 | ## How It Works: The System Behind Personalized AI Openers 20 | 21 | Here's the high-level workflow of what we're building: 22 | 23 | 1. **Input**: You provide basic information about your prospect (name, relevant keywords) 24 | 2. **Research**: The AI searches the web for information about your prospect 25 | 3. **Analysis**: The AI analyzes the search results for personalization opportunities 26 | 4. **Generation**: The AI crafts a personalized opening message based on its research 27 | 5. **Output**: You get a ready-to-use opening message 28 | 29 | The entire process takes about 30-60 seconds per prospect—compared to the 15+ minutes it might take to do this research manually. 30 | 31 | This system is built using [Pocket Flow](https://github.com/the-pocket/PocketFlow), a 100-line minimalist framework for building LLM applications. What makes Pocket Flow special isn't just its compact size, but how it reveals the inner workings of AI application development in a clear, educational way. 32 | 33 | ## Getting Started: Setting Up Your Environment 34 | 35 | To follow along with this tutorial, you'll need: 36 | 37 | 1. API keys for AI and search services 38 | 2. Basic Python knowledge 39 | 3. Git to clone the repository 40 | 41 | > **Note:** The implementation uses Google Search API and Claude for AI, but you can easily replace them with your preferred services such as OpenAI GPT or SerpAPI depending on your needs. 42 | 43 | If you just want to try it out first, you can use the [live demo](https://pocket-opener-851564657364.us-east1.run.app/). 44 | 45 | ### Step 1: Clone the Repository 46 | 47 | Start by cloning the repository with all the code you need: 48 | 49 | ```bash 50 | git clone https://github.com/The-Pocket/Tutorial-Cold-Email-Personalization.git 51 | cd Tutorial-Cold-Email-Personalization 52 | ``` 53 | 54 | ### Step 2: Set Up Your API Keys 55 | 56 | Create a `.env` file in the project root directory with your API keys: 57 | 58 | ``` 59 | OPENAI_API_KEY=your_openai_api_key_here 60 | ``` 61 | 62 | The tool is designed to work with different AI and search providers. Here's a simple implementation of `call_llm` using OpenAI: 63 | 64 | ```python 65 | # utils/call_llm.py example 66 | import os 67 | from openai import OpenAI 68 | 69 | def call_llm(prompt): 70 | """Simple implementation using OpenAI.""" 71 | client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) 72 | response = client.chat.completions.create( 73 | model="gpt-3.5-turbo", 74 | messages=[{"role": "user", "content": prompt}] 75 | ) 76 | return response.choices[0].message.content 77 | 78 | # Test the function 79 | if __name__ == "__main__": 80 | print(call_llm("Write a one-sentence greeting.")) 81 | ``` 82 | 83 | You can easily modify this to use other AI services or add features like caching. 84 | 85 | The `search_web` utility function is implemented in a similar way—a simple function that takes a query and returns search results. Just like with the LLM implementation, you can swap in your preferred search provider (Google Search, SerpAPI, etc.) based on your needs. 86 | 87 | Make sure your API keys work by testing the utility functions: 88 | 89 | ```bash 90 | python utils/call_llm.py # Test your AI implementation 91 | python utils/search_web.py # Test your search implementation 92 | ``` 93 | 94 | If both scripts run without errors, you're ready to go! 95 | 96 | ### Step 3: Install Dependencies 97 | 98 | Install the required Python packages: 99 | 100 | ```bash 101 | pip install -r requirements.txt 102 | ``` 103 | 104 | ## Using the Tool: Your First Personalized Opener 105 | 106 | Now that you have everything set up, let's generate your first personalized opener. The tool offers multiple interfaces to fit different workflows: 107 | 108 | - **Command line interface** for quick individual messages 109 | - **Web UI** for a user-friendly interactive experience 110 | - **Batch processing** for handling multiple prospects at scale 111 | 112 | Choose the method that works best for your specific needs: 113 | 114 | ### Method 1: Using the Command Line Interface 115 | 116 | The simplest way to generate a single opener is through the command line: 117 | 118 | ```bash 119 | python main.py 120 | ``` 121 | 122 | This will prompt you for: 123 | - First name 124 | - Last name 125 | - Keywords related to the person (like company names or topics they're known for) 126 | 127 | ### Method 2: Using the Web Interface 128 | 129 | For a more user-friendly experience, run the web interface: 130 | 131 | ```bash 132 | streamlit run app.py 133 | ``` 134 | 135 | This will open a browser window where you can: 136 | 1. Enter the target person's information 137 | 2. Define personalization factors to look for 138 | 3. Set your preferred message style 139 | 4. Generate and review the opening message 140 | 141 | ### Method 3: Batch Processing from CSV 142 | 143 | For efficiently handling multiple prospects at once, the tool provides a powerful batch processing mode: 144 | 145 | ```bash 146 | python main_batch.py --input my_targets.csv --output my_results.csv 147 | ``` 148 | 149 | Your input CSV should have three columns: 150 | - `first_name`: Prospect's first name 151 | - `last_name`: Prospect's last name 152 | - `keywords`: Space-separated keywords (e.g., "Tesla SpaceX entrepreneur") 153 | 154 | This is particularly useful when you need to reach out to dozens or hundreds of prospects. The system will: 155 | 156 | 1. Process each row in your CSV file 157 | 2. Perform web searches for each prospect 158 | 3. Generate personalized openers for each one 159 | 4. Write the results back to your output CSV file 160 | 161 | The output CSV will contain all your original data plus an additional column with the generated opening message for each prospect. You can then import this directly into your email marketing tool or CRM system. 162 | 163 | Example batch processing workflow: 164 | 165 | 1. Prepare a CSV with your prospect list 166 | 2. Run the batch processing command 167 | 3. Let it run (processing time: ~1 minute per prospect) 168 | 4. Review and refine the generated openers in the output CSV 169 | 5. Import into your outreach tool and start your campaign 170 | 171 | ### Recommended Workflow 172 | 173 | For the best results, we recommend this approach: 174 | 175 | 1. **Start with single mode or the Streamlit UI** to fine-tune your personalization factors and message style. This gives you immediate feedback on what works well. 176 | 2. **Experiment with different settings** for a few test prospects until you find the perfect combination of personalization factors and style preferences. 177 | 3. **Once satisfied with the results**, scale up using the batch processing mode to handle your entire prospect list. 178 | 179 | This workflow ensures you don't waste time and API calls processing a large batch with suboptimal settings, and helps you refine your approach before scaling. 180 | 181 | ## Understanding the Magic: How the AI Personalization Works 182 | 183 | This system is built using [Pocket Flow](https://github.com/the-pocket/PocketFlow), a 100-line minimalist framework for building LLM applications. What makes Pocket Flow special isn't just its compact size, but how it reveals the inner workings of AI application development in a clear, educational way. 184 | 185 | Unlike complex frameworks that hide implementation details, Pocket Flow's minimalist design makes it perfect for learning how LLM applications actually work under the hood. With just 100 lines of core code, it's impressively expressive, allowing you to build sophisticated AI workflows while still understanding every component. Despite its small size, it provides many of the same capabilities you'd find in larger libraries like LangChain, LangGraph, or CrewAI: 186 | 187 | - **Agents & Tools**: Build autonomous AI agents that can use tools and make decisions 188 | - **RAG (Retrieval Augmented Generation)**: Enhance LLM responses with external knowledge 189 | - **Task Decomposition**: Break complex tasks into manageable subtasks 190 | - **Parallel Processing**: Handle multiple tasks efficiently with batch processing 191 | - **Multi-Agent Systems**: Coordinate multiple AI agents working together 192 | 193 | The difference? You can read and understand Pocket Flow's entire codebase in minutes, making it perfect for learning and customization. 194 | 195 | Pocket Flow's approach to complex AI workflows is elegant and transparent: 196 | 197 | - **Graph-based Processing**: Each task is a node in a graph, making the flow easy to understand and modify 198 | - **Shared State**: Nodes communicate through a shared store, eliminating complex data passing 199 | - **Batch Processing**: Built-in support for parallel processing of multiple items 200 | - **Flexibility**: Easy to swap components or add new features without breaking existing code 201 | 202 | Let's look at how we've structured our cold outreach system using Pocket Flow: 203 | 204 | ```mermaid 205 | flowchart LR 206 | A[SearchPersonNode] --> B[ContentRetrievalNode] 207 | B --> C[AnalyzeResultsBatchNode] 208 | C --> D[DraftOpeningNode] 209 | 210 | classDef batch fill:#f9f,stroke:#333,stroke-width:2px 211 | class B,C batch 212 | ``` 213 | 214 | The system follows a straightforward flow pattern with these core components: 215 | 216 | 1. **SearchPersonNode**: Searches the web for information about the prospect 217 | 2. **ContentRetrievalNode** (Batch): Retrieves and processes content from search results in parallel 218 | 3. **AnalyzeResultsBatchNode** (Batch): Analyzes content for personalization opportunities using LLM 219 | 4. **DraftOpeningNode**: Creates the final personalized opener 220 | 221 | What makes this architecture powerful is its: 222 | - **Modularity**: Each component can be improved independently 223 | - **Parallel Processing**: Batch nodes handle multiple items simultaneously 224 | - **Flexibility**: You can swap in different search providers or LLMs 225 | - **Scalability**: Works for single prospects or batch processing 226 | 227 | Now, let's break down the implementation details for each phase: 228 | 229 | ### 1. Web Search Phase 230 | 231 | The system first searches the web for information about your prospect using their name and the keywords you provided: 232 | 233 | ```python 234 | # From flow.py 235 | class SearchPersonNode(Node): 236 | def prep(self, shared): 237 | first_name = shared["input"]["first_name"] 238 | last_name = shared["input"]["last_name"] 239 | keywords = shared["input"]["keywords"] 240 | 241 | query = f"{first_name} {last_name} {keywords}" 242 | return query 243 | 244 | def exec(self, query): 245 | search_results = search_web(query) 246 | return search_results 247 | ``` 248 | 249 | By default, the implementation uses Google Search API, but you can easily swap this out for another search provider like SerpAPI in the `search_web` utility function. This flexibility allows you to use whichever search provider works best for your needs or budget. 250 | 251 | ### 2. Content Retrieval Phase 252 | 253 | Next, it retrieves and processes the content from the top search results: 254 | 255 | ```python 256 | class ContentRetrievalNode(BatchNode): 257 | def prep(self, shared): 258 | search_results = shared["search_results"] 259 | urls = [result["link"] for result in search_results if "link" in result] 260 | return urls 261 | 262 | def exec(self, url): 263 | content = get_html_content(url) 264 | return {"url": url, "content": content} 265 | ``` 266 | 267 | ### 3. Analysis Phase 268 | 269 | The system then analyzes the content looking for specific personalization factors you defined: 270 | 271 | ```python 272 | class AnalyzeResultsBatchNode(BatchNode): 273 | def exec(self, url_content_pair): 274 | # Prepare prompt for LLM analysis 275 | prompt = f"""Analyze the following webpage content about {self.first_name} {self.last_name}. 276 | Look for the following personalization factors: 277 | {self._format_personalization_factors(self.personalization_factors)}""" 278 | 279 | # LLM analyzes the content for personalization opportunities 280 | analysis_results = call_llm(prompt) 281 | return analysis_results 282 | ``` 283 | 284 | ### 4. Generation Phase 285 | 286 | Finally, the system crafts a personalized opener based on the discovered information: 287 | 288 | ```python 289 | class DraftOpeningNode(Node): 290 | def exec(self, prep_data): 291 | first_name, last_name, style, personalization = prep_data 292 | 293 | prompt = f"""Draft a personalized opening message for a cold outreach email to {first_name} {last_name}. 294 | 295 | Style preferences: {style} 296 | 297 | Personalization details: 298 | {self._format_personalization_details(personalization)} 299 | 300 | Only write the opening message. Be specific, authentic, and concise.""" 301 | 302 | opening_message = call_llm(prompt) 303 | return opening_message 304 | ``` 305 | 306 | The system uses the `call_llm` utility function which can be configured to use different AI models like Claude or GPT models from OpenAI. This allows you to experiment with different LLMs to find the one that creates the most effective openers for your specific use case. 307 | 308 | ## Customizing for Your Needs 309 | 310 | The real power of this system is in the personalization factors you define. Here are some effective examples: 311 | 312 | ### For Job Seekers: 313 | - **Recent company news**: "I saw [Company] just announced [News]. I'd love to discuss how my experience in [Skill] could help with this initiative." 314 | - **Shared alma mater**: "As a fellow [University] alum, I was excited to see your work on [Project]." 315 | - **Mutual connection**: "I noticed we're both connected to [Name]. I've worked with them on [Project] and they spoke highly of your team." 316 | 317 | ### For Sales Professionals: 318 | - **Pain points**: "I noticed from your recent interview that [Company] is facing challenges with [Problem]. We've helped similar companies solve this by..." 319 | - **Growth initiatives**: "Congratulations on your expansion into [Market]. Our solution has helped similar companies accelerate growth in this area by..." 320 | - **Competitor mentions**: "I saw you mentioned working with [Competitor] in the past. Many of our clients who switched from them found our approach to [Feature] more effective because..." 321 | 322 | ### For Founders: 323 | - **Investment thesis alignment**: "Your recent investment in [Company] caught my attention. Our startup is also focused on [Similar Space], but with a unique approach to..." 324 | - **Industry challenges**: "I read your thoughts on [Industry Challenge] in [Publication]. We're building a solution that addresses this exact issue by..." 325 | - **Shared vision**: "Your talk at [Conference] about [Topic] resonated with me. We're building technology that aligns with your vision of [Vision]..." 326 | 327 | ## Tips for Better Results 328 | 329 | Here are some tips for getting the best results from the system: 330 | 331 | 1. **Be specific with keywords**: Instead of just "CEO", try "CEO FinTech YCombinator" 332 | 2. **Test different personalization factors**: Some work better than others depending on the person 333 | 3. **Refine your style preferences**: The more specific your style guidance, the better the results 334 | 4. **Review and edit**: AI-generated openers are a starting point, not the final product 335 | 5. **A/B test**: Try different approaches and track which ones get better responses 336 | 337 | ## Conclusion: Beyond Cold Outreach 338 | 339 | While we've focused on cold outreach openers, the same approach can be used for: 340 | 341 | - Personalizing follow-ups after meetings 342 | - Crafting tailored proposals based on prospect research 343 | - Creating customized content that resonates with specific audience segments 344 | - Building detailed prospect profiles for your sales team 345 | 346 | The possibilities are endless when you combine AI with thoughtful personalization strategies. 347 | 348 | The key is striking the right balance: using AI to scale your outreach without losing the human touch that makes connections meaningful. 349 | 350 | --- 351 | 352 | Want to explore the full code? Check out the [GitHub repository](https://github.com/The-Pocket/Tutorial-Cold-Email-Personalization). 353 | 354 | Have questions or want to share your results? Leave a comment below! 355 | 356 | -------------------------------------------------------------------------------- /.cursorrules: -------------------------------------------------------------------------------- 1 | ================================================ 2 | File: docs/guide.md 3 | ================================================ 4 | --- 5 | layout: default 6 | title: "Build your LLM App" 7 | --- 8 | 9 | # LLM Application Development Playbook 10 | 11 | > If you are an AI assistant involved in building LLM Apps, read this guide **VERY, VERY** carefully! This is the most important chapter in the entire document. Throughout development, you should always (1) start with a small and simple solution, (2) design at a high level (`docs/design.md`) before implementation, and (3) frequently ask humans for feedback and clarification. 12 | {: .warning } 13 | 14 | ## System Design Steps 15 | 16 | These system designs should be a collaboration between humans and AI assistants: 17 | 18 | | Stage | Human | AI | Comment | 19 | |:-----------------------|:----------:|:---------:|:------------------------------------------------------------------------| 20 | | 1. Project Requirements | ★★★ High | ★☆☆ Low | Humans understand the requirements and context best. | 21 | | 2. Utility Functions | ★★☆ Medium | ★★☆ Medium | The human is familiar with external APIs and integrations, and the AI assists with implementation. | 22 | | 3. Flow Design | ★★☆ Medium | ★★☆ Medium | The human identifies complex and ambiguous parts, and the AI helps with redesign. | 23 | | 4. Data Schema | ★☆☆ Low | ★★★ High | The AI assists in designing the data schema based on the flow. | 24 | | 5. Implementation | ★☆☆ Low | ★★★ High | The human identifies complex and ambiguous parts, and the AI helps with redesign. | 25 | | 6. Optimization | ★★☆ Medium | ★★☆ Medium | The human reviews the code and evaluates the results, while the AI helps optimize. | 26 | | 7. Reliability | ★☆☆ Low | ★★★ High | The AI helps write test cases and address corner cases. | 27 | 28 | 1. **Project Requirements**: Clarify the requirements for your project, and evaluate whether an AI system is a good fit. An AI systems are: 29 | - suitable for routine tasks that require common sense (e.g., filling out forms, replying to emails). 30 | - suitable for creative tasks where all inputs are provided (e.g., building slides, writing SQL). 31 | - **NOT** suitable for tasks that are highly ambiguous and require complex information (e.g., building a startup). 32 | - > **If a human can’t solve it, an LLM can’t automate it!** Before building an LLM system, thoroughly understand the problem by manually solving example inputs to develop intuition. 33 | {: .best-practice } 34 | 35 | 2. **Utility Functions**: AI system is the decision-maker and relies on *external utility functions* to: 36 | 37 |
38 | 39 | - Read inputs (e.g., retrieving Slack messages, reading emails) 40 | - Write outputs (e.g., generating reports, sending emails) 41 | - Use external tools (e.g., calling LLMs, searching the web) 42 | - In contrast, *LLM-based tasks* (e.g., summarizing text, analyzing sentiment) are **NOT** utility functions. Instead, they are *internal core functions* within the AI system—designed in step 3—and are built on top of the utility functions. 43 | - > **Start small!** Only include the most important ones to begin with! 44 | {: .best-practice } 45 | 46 | 3. **Flow Design (Compute)**: Create a high-level outline for your application’s flow. 47 | - Identify potential design patterns (e.g., Batch, Agent, RAG). 48 | - For each node, specify: 49 | - **Purpose**: The high-level compute logic 50 | - **Type**: Regular node, Batch node, async node, or another type 51 | - `exec`: The specific utility function to call (ideally, one function per node) 52 | 53 | 4. **Data Schema (Data)**: Plan how data will be stored and updated. 54 | - For simple apps, use an in-memory dictionary. 55 | - For more complex apps or when persistence is required, use a database. 56 | - For each node, specify: 57 | - `prep`: How the node reads data 58 | - `post`: How the node writes data 59 | 60 | 5. **Implementation**: Implement nodes and flows based on the design. 61 | - Start with a simple, direct approach (avoid over-engineering and full-scale type checking or testing). Let it fail fast to identify weaknesses. 62 | - Add logging throughout the code to facilitate debugging. 63 | 64 | 6. **Optimization**: 65 | - **Use Intuition**: For a quick initial evaluation, human intuition is often a good start. 66 | - **Redesign Flow (Back to Step 3)**: Consider breaking down tasks further, introducing agentic decisions, or better managing input contexts. 67 | - If your flow design is already solid, move on to micro-optimizations: 68 | - **Prompt Engineering**: Use clear, specific instructions with examples to reduce ambiguity. 69 | - **In-Context Learning**: Provide robust examples for tasks that are difficult to specify with instructions alone. 70 | 71 | - > **You’ll likely iterate a lot!** Expect to repeat Steps 3–6 hundreds of times. 72 | > 73 | >
74 | {: .best-practice } 75 | 76 | 7. **Reliability** 77 | - **Node Retries**: Add checks in the node `exec` to ensure outputs meet requirements, and consider increasing `max_retries` and `wait` times. 78 | - **Logging and Visualization**: Maintain logs of all attempts and visualize node results for easier debugging. 79 | - **Self-Evaluation**: Add a separate node (powered by an LLM) to review outputs when results are uncertain. 80 | 81 | ## Example LLM Project File Structure 82 | 83 | ``` 84 | my_project/ 85 | ├── main.py 86 | ├── flow.py 87 | ├── utils/ 88 | │ ├── __init__.py 89 | │ ├── call_llm.py 90 | │ └── search_web.py 91 | ├── requirements.txt 92 | └── docs/ 93 | └── design.md 94 | ``` 95 | 96 | - **`docs/design.md`**: Contains project documentation for each step above. This should be high-level and no-code. 97 | - **`utils/`**: Contains all utility functions. 98 | - It’s recommended to dedicate one Python file to each API call, for example `call_llm.py` or `search_web.py`. 99 | - Each file should also include a `main()` function to try that API call 100 | - **`flow.py`**: Implements the application’s flow, starting with node definitions followed by the overall structure. 101 | - **`main.py`**: Serves as the project’s entry point. 102 | 103 | ================================================ 104 | File: docs/index.md 105 | ================================================ 106 | --- 107 | layout: default 108 | title: "Home" 109 | nav_order: 1 110 | --- 111 | 112 | # Pocket Flow 113 | 114 | A [100-line](https://github.com/the-pocket/PocketFlow/blob/main/pocketflow/__init__.py) minimalist LLM framework for *Agents, Task Decomposition, RAG, etc*. 115 | 116 | We model the LLM workflow as a **Graph + Shared Store**: 117 | - **Nodes** handle simple (LLM) tasks. 118 | - Nodes connect through **Actions** (labeled edges) for *Agents*. 119 | - **Flows** orchestrate a directed graph of Nodes for *Task Decomposition*. 120 | - Nodes within Flows communicate through a **Shared Store**. 121 | - A Flow can be used as a Node (for **Nesting**). 122 | - **Batch** Nodes/Flows for data-intensive tasks. 123 | - **Async** Nodes/Flows allow waits for asynchronous tasks. 124 | 125 |
126 | 127 |
128 | 129 | > If you are an AI assistant reading this document, please read all the non-optional and non-advanced parts. If you are helping a human build an LLM app, please carefully review the [Development Playbook](./guide.md). 130 | {: .warning } 131 | 132 | ## Core Abstraction 133 | 134 | - [Node](./core_abstraction/node.md) 135 | - [Flow](./core_abstraction/flow.md) 136 | - [Communication](./core_abstraction/communication.md) 137 | - [Batch](./core_abstraction/batch.md) 138 | - [(Advanced) Async](./core_abstraction/async.md) 139 | - [(Advanced) Parallel](./core_abstraction/parallel.md) 140 | 141 | ## Utility Function 142 | 143 | - [LLM Wrapper](./utility_function/llm.md) 144 | - [Tool](./utility_function/tool.md) 145 | - [(Optional) Viz and Debug](./utility_function/viz.md) 146 | - Chunking 147 | 148 | > We do not provide built-in utility functions. Example implementations are provided as reference. 149 | {: .warning } 150 | 151 | 152 | ## Design Pattern 153 | 154 | - [Structured Output](./design_pattern/structure.md) 155 | - [Workflow](./design_pattern/workflow.md) 156 | - [Map Reduce](./design_pattern/mapreduce.md) 157 | - [RAG](./design_pattern/rag.md) 158 | - [Agent](./design_pattern/agent.md) 159 | - [(Optional) Chat Memory](./design_pattern/memory.md) 160 | - [(Advanced) Multi-Agents](./design_pattern/multi_agent.md) 161 | - Evaluation 162 | 163 | ## [Develop your LLM Apps](./guide.md) 164 | 165 | ================================================ 166 | File: docs/core_abstraction/async.md 167 | ================================================ 168 | --- 169 | layout: default 170 | title: "(Advanced) Async" 171 | parent: "Core Abstraction" 172 | nav_order: 5 173 | --- 174 | 175 | # (Advanced) Async 176 | 177 | **Async** Nodes implement `prep_async()`, `exec_async()`, `exec_fallback_async()`, and/or `post_async()`. This is useful for: 178 | 179 | 1. **prep_async()**: For *fetching/reading data (files, APIs, DB)* in an I/O-friendly way. 180 | 2. **exec_async()**: Typically used for async LLM calls. 181 | 3. **post_async()**: For *awaiting user feedback*, *coordinating across multi-agents* or any additional async steps after `exec_async()`. 182 | 183 | **Note**: `AsyncNode` must be wrapped in `AsyncFlow`. `AsyncFlow` can also include regular (sync) nodes. 184 | 185 | ### Example 186 | 187 | ```python 188 | class SummarizeThenVerify(AsyncNode): 189 | async def prep_async(self, shared): 190 | # Example: read a file asynchronously 191 | doc_text = await read_file_async(shared["doc_path"]) 192 | return doc_text 193 | 194 | async def exec_async(self, prep_res): 195 | # Example: async LLM call 196 | summary = await call_llm_async(f"Summarize: {prep_res}") 197 | return summary 198 | 199 | async def post_async(self, shared, prep_res, exec_res): 200 | # Example: wait for user feedback 201 | decision = await gather_user_feedback(exec_res) 202 | if decision == "approve": 203 | shared["summary"] = exec_res 204 | return "approve" 205 | return "deny" 206 | 207 | summarize_node = SummarizeThenVerify() 208 | final_node = Finalize() 209 | 210 | # Define transitions 211 | summarize_node - "approve" >> final_node 212 | summarize_node - "deny" >> summarize_node # retry 213 | 214 | flow = AsyncFlow(start=summarize_node) 215 | 216 | async def main(): 217 | shared = {"doc_path": "document.txt"} 218 | await flow.run_async(shared) 219 | print("Final Summary:", shared.get("summary")) 220 | 221 | asyncio.run(main()) 222 | ``` 223 | 224 | ================================================ 225 | File: docs/core_abstraction/batch.md 226 | ================================================ 227 | --- 228 | layout: default 229 | title: "Batch" 230 | parent: "Core Abstraction" 231 | nav_order: 4 232 | --- 233 | 234 | # Batch 235 | 236 | **Batch** makes it easier to handle large inputs in one Node or **rerun** a Flow multiple times. Example use cases: 237 | - **Chunk-based** processing (e.g., splitting large texts). 238 | - **Iterative** processing over lists of input items (e.g., user queries, files, URLs). 239 | 240 | ## 1. BatchNode 241 | 242 | A **BatchNode** extends `Node` but changes `prep()` and `exec()`: 243 | 244 | - **`prep(shared)`**: returns an **iterable** (e.g., list, generator). 245 | - **`exec(item)`**: called **once** per item in that iterable. 246 | - **`post(shared, prep_res, exec_res_list)`**: after all items are processed, receives a **list** of results (`exec_res_list`) and returns an **Action**. 247 | 248 | 249 | ### Example: Summarize a Large File 250 | 251 | ```python 252 | class MapSummaries(BatchNode): 253 | def prep(self, shared): 254 | # Suppose we have a big file; chunk it 255 | content = shared["data"] 256 | chunk_size = 10000 257 | chunks = [content[i:i+chunk_size] for i in range(0, len(content), chunk_size)] 258 | return chunks 259 | 260 | def exec(self, chunk): 261 | prompt = f"Summarize this chunk in 10 words: {chunk}" 262 | summary = call_llm(prompt) 263 | return summary 264 | 265 | def post(self, shared, prep_res, exec_res_list): 266 | combined = "\n".join(exec_res_list) 267 | shared["summary"] = combined 268 | return "default" 269 | 270 | map_summaries = MapSummaries() 271 | flow = Flow(start=map_summaries) 272 | flow.run(shared) 273 | ``` 274 | 275 | --- 276 | 277 | ## 2. BatchFlow 278 | 279 | A **BatchFlow** runs a **Flow** multiple times, each time with different `params`. Think of it as a loop that replays the Flow for each parameter set. 280 | 281 | 282 | ### Example: Summarize Many Files 283 | 284 | ```python 285 | class SummarizeAllFiles(BatchFlow): 286 | def prep(self, shared): 287 | # Return a list of param dicts (one per file) 288 | filenames = list(shared["data"].keys()) # e.g., ["file1.txt", "file2.txt", ...] 289 | return [{"filename": fn} for fn in filenames] 290 | 291 | # Suppose we have a per-file Flow (e.g., load_file >> summarize >> reduce): 292 | summarize_file = SummarizeFile(start=load_file) 293 | 294 | # Wrap that flow into a BatchFlow: 295 | summarize_all_files = SummarizeAllFiles(start=summarize_file) 296 | summarize_all_files.run(shared) 297 | ``` 298 | 299 | ### Under the Hood 300 | 1. `prep(shared)` returns a list of param dicts—e.g., `[{filename: "file1.txt"}, {filename: "file2.txt"}, ...]`. 301 | 2. The **BatchFlow** loops through each dict. For each one: 302 | - It merges the dict with the BatchFlow’s own `params`. 303 | - It calls `flow.run(shared)` using the merged result. 304 | 3. This means the sub-Flow is run **repeatedly**, once for every param dict. 305 | 306 | --- 307 | 308 | ## 3. Nested or Multi-Level Batches 309 | 310 | You can nest a **BatchFlow** in another **BatchFlow**. For instance: 311 | - **Outer** batch: returns a list of diretory param dicts (e.g., `{"directory": "/pathA"}`, `{"directory": "/pathB"}`, ...). 312 | - **Inner** batch: returning a list of per-file param dicts. 313 | 314 | At each level, **BatchFlow** merges its own param dict with the parent’s. By the time you reach the **innermost** node, the final `params` is the merged result of **all** parents in the chain. This way, a nested structure can keep track of the entire context (e.g., directory + file name) at once. 315 | 316 | ```python 317 | 318 | class FileBatchFlow(BatchFlow): 319 | def prep(self, shared): 320 | directory = self.params["directory"] 321 | # e.g., files = ["file1.txt", "file2.txt", ...] 322 | files = [f for f in os.listdir(directory) if f.endswith(".txt")] 323 | return [{"filename": f} for f in files] 324 | 325 | class DirectoryBatchFlow(BatchFlow): 326 | def prep(self, shared): 327 | directories = [ "/path/to/dirA", "/path/to/dirB"] 328 | return [{"directory": d} for d in directories] 329 | 330 | # MapSummaries have params like {"directory": "/path/to/dirA", "filename": "file1.txt"} 331 | inner_flow = FileBatchFlow(start=MapSummaries()) 332 | outer_flow = DirectoryBatchFlow(start=inner_flow) 333 | ``` 334 | 335 | ================================================ 336 | File: docs/core_abstraction/communication.md 337 | ================================================ 338 | --- 339 | layout: default 340 | title: "Communication" 341 | parent: "Core Abstraction" 342 | nav_order: 3 343 | --- 344 | 345 | # Communication 346 | 347 | Nodes and Flows **communicate** in two ways: 348 | 349 | 1. **Shared Store (recommended)** 350 | 351 | - A global data structure (often an in-mem dict) that all nodes can read and write by `prep()` and `post()`. 352 | - Great for data results, large content, or anything multiple nodes need. 353 | - You shall design the data structure and populate it ahead. 354 | 355 | 2. **Params (only for [Batch](./batch.md))** 356 | - Each node has a local, ephemeral `params` dict passed in by the **parent Flow**, used as an identifier for tasks. Parameter keys and values shall be **immutable**. 357 | - Good for identifiers like filenames or numeric IDs, in Batch mode. 358 | 359 | If you know memory management, think of the **Shared Store** like a **heap** (shared by all function calls), and **Params** like a **stack** (assigned by the caller). 360 | 361 | > Use `Shared Store` for almost all cases. It's flexible and easy to manage. It separates *Data Schema* from *Compute Logic*, making the code easier to maintain. `Params` is more a syntax sugar for [Batch](./batch.md). 362 | {: .best-practice } 363 | 364 | --- 365 | 366 | ## 1. Shared Store 367 | 368 | ### Overview 369 | 370 | A shared store is typically an in-mem dictionary, like: 371 | ```python 372 | shared = {"data": {}, "summary": {}, "config": {...}, ...} 373 | ``` 374 | 375 | It can also contain local file handlers, DB connections, or a combination for persistence. We recommend deciding the data structure or DB schema first based on your app requirements. 376 | 377 | ### Example 378 | 379 | ```python 380 | class LoadData(Node): 381 | def post(self, shared, prep_res, exec_res): 382 | # We write data to shared store 383 | shared["data"] = "Some text content" 384 | return None 385 | 386 | class Summarize(Node): 387 | def prep(self, shared): 388 | # We read data from shared store 389 | return shared["data"] 390 | 391 | def exec(self, prep_res): 392 | # Call LLM to summarize 393 | prompt = f"Summarize: {prep_res}" 394 | summary = call_llm(prompt) 395 | return summary 396 | 397 | def post(self, shared, prep_res, exec_res): 398 | # We write summary to shared store 399 | shared["summary"] = exec_res 400 | return "default" 401 | 402 | load_data = LoadData() 403 | summarize = Summarize() 404 | load_data >> summarize 405 | flow = Flow(start=load_data) 406 | 407 | shared = {} 408 | flow.run(shared) 409 | ``` 410 | 411 | Here: 412 | - `LoadData` writes to `shared["data"]`. 413 | - `Summarize` reads from `shared["data"]`, summarizes, and writes to `shared["summary"]`. 414 | 415 | --- 416 | 417 | ## 2. Params 418 | 419 | **Params** let you store *per-Node* or *per-Flow* config that doesn't need to live in the shared store. They are: 420 | - **Immutable** during a Node's run cycle (i.e., they don't change mid-`prep->exec->post`). 421 | - **Set** via `set_params()`. 422 | - **Cleared** and updated each time a parent Flow calls it. 423 | 424 | > Only set the uppermost Flow params because others will be overwritten by the parent Flow. 425 | > 426 | > If you need to set child node params, see [Batch](./batch.md). 427 | {: .warning } 428 | 429 | Typically, **Params** are identifiers (e.g., file name, page number). Use them to fetch the task you assigned or write to a specific part of the shared store. 430 | 431 | ### Example 432 | 433 | ```python 434 | # 1) Create a Node that uses params 435 | class SummarizeFile(Node): 436 | def prep(self, shared): 437 | # Access the node's param 438 | filename = self.params["filename"] 439 | return shared["data"].get(filename, "") 440 | 441 | def exec(self, prep_res): 442 | prompt = f"Summarize: {prep_res}" 443 | return call_llm(prompt) 444 | 445 | def post(self, shared, prep_res, exec_res): 446 | filename = self.params["filename"] 447 | shared["summary"][filename] = exec_res 448 | return "default" 449 | 450 | # 2) Set params 451 | node = SummarizeFile() 452 | 453 | # 3) Set Node params directly (for testing) 454 | node.set_params({"filename": "doc1.txt"}) 455 | node.run(shared) 456 | 457 | # 4) Create Flow 458 | flow = Flow(start=node) 459 | 460 | # 5) Set Flow params (overwrites node params) 461 | flow.set_params({"filename": "doc2.txt"}) 462 | flow.run(shared) # The node summarizes doc2, not doc1 463 | ``` 464 | 465 | ================================================ 466 | File: docs/core_abstraction/flow.md 467 | ================================================ 468 | --- 469 | layout: default 470 | title: "Flow" 471 | parent: "Core Abstraction" 472 | nav_order: 2 473 | --- 474 | 475 | # Flow 476 | 477 | A **Flow** orchestrates a graph of Nodes. You can chain Nodes in a sequence or create branching depending on the **Actions** returned from each Node's `post()`. 478 | 479 | ## 1. Action-based Transitions 480 | 481 | Each Node's `post()` returns an **Action** string. By default, if `post()` doesn't return anything, we treat that as `"default"`. 482 | 483 | You define transitions with the syntax: 484 | 485 | 1. **Basic default transition**: `node_a >> node_b` 486 | This means if `node_a.post()` returns `"default"`, go to `node_b`. 487 | (Equivalent to `node_a - "default" >> node_b`) 488 | 489 | 2. **Named action transition**: `node_a - "action_name" >> node_b` 490 | This means if `node_a.post()` returns `"action_name"`, go to `node_b`. 491 | 492 | It's possible to create loops, branching, or multi-step flows. 493 | 494 | ## 2. Creating a Flow 495 | 496 | A **Flow** begins with a **start** node. You call `Flow(start=some_node)` to specify the entry point. When you call `flow.run(shared)`, it executes the start node, looks at its returned Action from `post()`, follows the transition, and continues until there's no next node. 497 | 498 | ### Example: Simple Sequence 499 | 500 | Here's a minimal flow of two nodes in a chain: 501 | 502 | ```python 503 | node_a >> node_b 504 | flow = Flow(start=node_a) 505 | flow.run(shared) 506 | ``` 507 | 508 | - When you run the flow, it executes `node_a`. 509 | - Suppose `node_a.post()` returns `"default"`. 510 | - The flow then sees `"default"` Action is linked to `node_b` and runs `node_b`. 511 | - `node_b.post()` returns `"default"` but we didn't define `node_b >> something_else`. So the flow ends there. 512 | 513 | ### Example: Branching & Looping 514 | 515 | Here's a simple expense approval flow that demonstrates branching and looping. The `ReviewExpense` node can return three possible Actions: 516 | 517 | - `"approved"`: expense is approved, move to payment processing 518 | - `"needs_revision"`: expense needs changes, send back for revision 519 | - `"rejected"`: expense is denied, finish the process 520 | 521 | We can wire them like this: 522 | 523 | ```python 524 | # Define the flow connections 525 | review - "approved" >> payment # If approved, process payment 526 | review - "needs_revision" >> revise # If needs changes, go to revision 527 | review - "rejected" >> finish # If rejected, finish the process 528 | 529 | revise >> review # After revision, go back for another review 530 | payment >> finish # After payment, finish the process 531 | 532 | flow = Flow(start=review) 533 | ``` 534 | 535 | Let's see how it flows: 536 | 537 | 1. If `review.post()` returns `"approved"`, the expense moves to the `payment` node 538 | 2. If `review.post()` returns `"needs_revision"`, it goes to the `revise` node, which then loops back to `review` 539 | 3. If `review.post()` returns `"rejected"`, it moves to the `finish` node and stops 540 | 541 | ```mermaid 542 | flowchart TD 543 | review[Review Expense] -->|approved| payment[Process Payment] 544 | review -->|needs_revision| revise[Revise Report] 545 | review -->|rejected| finish[Finish Process] 546 | 547 | revise --> review 548 | payment --> finish 549 | ``` 550 | 551 | ### Running Individual Nodes vs. Running a Flow 552 | 553 | - `node.run(shared)`: Just runs that node alone (calls `prep->exec->post()`), returns an Action. 554 | - `flow.run(shared)`: Executes from the start node, follows Actions to the next node, and so on until the flow can't continue. 555 | 556 | > `node.run(shared)` **does not** proceed to the successor. 557 | > This is mainly for debugging or testing a single node. 558 | > 559 | > Always use `flow.run(...)` in production to ensure the full pipeline runs correctly. 560 | {: .warning } 561 | 562 | ## 3. Nested Flows 563 | 564 | A **Flow** can act like a Node, which enables powerful composition patterns. This means you can: 565 | 566 | 1. Use a Flow as a Node within another Flow's transitions. 567 | 2. Combine multiple smaller Flows into a larger Flow for reuse. 568 | 3. Node `params` will be a merging of **all** parents' `params`. 569 | 570 | ### Flow's Node Methods 571 | 572 | A **Flow** is also a **Node**, so it will run `prep()` and `post()`. However: 573 | 574 | - It **won't** run `exec()`, as its main logic is to orchestrate its nodes. 575 | - `post()` always receives `None` for `exec_res` and should instead get the flow execution results from the shared store. 576 | 577 | ### Basic Flow Nesting 578 | 579 | Here's how to connect a flow to another node: 580 | 581 | ```python 582 | # Create a sub-flow 583 | node_a >> node_b 584 | subflow = Flow(start=node_a) 585 | 586 | # Connect it to another node 587 | subflow >> node_c 588 | 589 | # Create the parent flow 590 | parent_flow = Flow(start=subflow) 591 | ``` 592 | 593 | When `parent_flow.run()` executes: 594 | 1. It starts `subflow` 595 | 2. `subflow` runs through its nodes (`node_a->node_b`) 596 | 3. After `subflow` completes, execution continues to `node_c` 597 | 598 | ### Example: Order Processing Pipeline 599 | 600 | Here's a practical example that breaks down order processing into nested flows: 601 | 602 | ```python 603 | # Payment processing sub-flow 604 | validate_payment >> process_payment >> payment_confirmation 605 | payment_flow = Flow(start=validate_payment) 606 | 607 | # Inventory sub-flow 608 | check_stock >> reserve_items >> update_inventory 609 | inventory_flow = Flow(start=check_stock) 610 | 611 | # Shipping sub-flow 612 | create_label >> assign_carrier >> schedule_pickup 613 | shipping_flow = Flow(start=create_label) 614 | 615 | # Connect the flows into a main order pipeline 616 | payment_flow >> inventory_flow >> shipping_flow 617 | 618 | # Create the master flow 619 | order_pipeline = Flow(start=payment_flow) 620 | 621 | # Run the entire pipeline 622 | order_pipeline.run(shared_data) 623 | ``` 624 | 625 | This creates a clean separation of concerns while maintaining a clear execution path: 626 | 627 | ```mermaid 628 | flowchart LR 629 | subgraph order_pipeline[Order Pipeline] 630 | subgraph paymentFlow["Payment Flow"] 631 | A[Validate Payment] --> B[Process Payment] --> C[Payment Confirmation] 632 | end 633 | 634 | subgraph inventoryFlow["Inventory Flow"] 635 | D[Check Stock] --> E[Reserve Items] --> F[Update Inventory] 636 | end 637 | 638 | subgraph shippingFlow["Shipping Flow"] 639 | G[Create Label] --> H[Assign Carrier] --> I[Schedule Pickup] 640 | end 641 | 642 | paymentFlow --> inventoryFlow 643 | inventoryFlow --> shippingFlow 644 | end 645 | ``` 646 | 647 | ================================================ 648 | File: docs/core_abstraction/node.md 649 | ================================================ 650 | --- 651 | layout: default 652 | title: "Node" 653 | parent: "Core Abstraction" 654 | nav_order: 1 655 | --- 656 | 657 | # Node 658 | 659 | A **Node** is the smallest building block. Each Node has 3 steps `prep->exec->post`: 660 | 661 |
662 | 663 |
664 | 665 | 1. `prep(shared)` 666 | - **Read and preprocess data** from `shared` store. 667 | - Examples: *query DB, read files, or serialize data into a string*. 668 | - Return `prep_res`, which is used by `exec()` and `post()`. 669 | 670 | 2. `exec(prep_res)` 671 | - **Execute compute logic**, with optional retries and error handling (below). 672 | - Examples: *(mostly) LLM calls, remote APIs, tool use*. 673 | - ⚠️ This shall be only for compute and **NOT** access `shared`. 674 | - ⚠️ If retries enabled, ensure idempotent implementation. 675 | - Return `exec_res`, which is passed to `post()`. 676 | 677 | 3. `post(shared, prep_res, exec_res)` 678 | - **Postprocess and write data** back to `shared`. 679 | - Examples: *update DB, change states, log results*. 680 | - **Decide the next action** by returning a *string* (`action = "default"` if *None*). 681 | 682 | > **Why 3 steps?** To enforce the principle of *separation of concerns*. The data storage and data processing are operated separately. 683 | > 684 | > All steps are *optional*. E.g., you can only implement `prep` and `post` if you just need to process data. 685 | {: .note } 686 | 687 | ### Fault Tolerance & Retries 688 | 689 | You can **retry** `exec()` if it raises an exception via two parameters when define the Node: 690 | 691 | - `max_retries` (int): Max times to run `exec()`. The default is `1` (**no** retry). 692 | - `wait` (int): The time to wait (in **seconds**) before next retry. By default, `wait=0` (no waiting). 693 | `wait` is helpful when you encounter rate-limits or quota errors from your LLM provider and need to back off. 694 | 695 | ```python 696 | my_node = SummarizeFile(max_retries=3, wait=10) 697 | ``` 698 | 699 | When an exception occurs in `exec()`, the Node automatically retries until: 700 | 701 | - It either succeeds, or 702 | - The Node has retried `max_retries - 1` times already and fails on the last attempt. 703 | 704 | You can get the current retry times (0-based) from `self.cur_retry`. 705 | 706 | ```python 707 | class RetryNode(Node): 708 | def exec(self, prep_res): 709 | print(f"Retry {self.cur_retry} times") 710 | raise Exception("Failed") 711 | ``` 712 | 713 | ### Graceful Fallback 714 | 715 | To **gracefully handle** the exception (after all retries) rather than raising it, override: 716 | 717 | ```python 718 | def exec_fallback(self, prep_res, exc): 719 | raise exc 720 | ``` 721 | 722 | By default, it just re-raises exception. But you can return a fallback result instead, which becomes the `exec_res` passed to `post()`. 723 | 724 | ### Example: Summarize file 725 | 726 | ```python 727 | class SummarizeFile(Node): 728 | def prep(self, shared): 729 | return shared["data"] 730 | 731 | def exec(self, prep_res): 732 | if not prep_res: 733 | return "Empty file content" 734 | prompt = f"Summarize this text in 10 words: {prep_res}" 735 | summary = call_llm(prompt) # might fail 736 | return summary 737 | 738 | def exec_fallback(self, prep_res, exc): 739 | # Provide a simple fallback instead of crashing 740 | return "There was an error processing your request." 741 | 742 | def post(self, shared, prep_res, exec_res): 743 | shared["summary"] = exec_res 744 | # Return "default" by not returning 745 | 746 | summarize_node = SummarizeFile(max_retries=3) 747 | 748 | # node.run() calls prep->exec->post 749 | # If exec() fails, it retries up to 3 times before calling exec_fallback() 750 | action_result = summarize_node.run(shared) 751 | 752 | print("Action returned:", action_result) # "default" 753 | print("Summary stored:", shared["summary"]) 754 | ``` 755 | 756 | ================================================ 757 | File: docs/core_abstraction/parallel.md 758 | ================================================ 759 | --- 760 | layout: default 761 | title: "(Advanced) Parallel" 762 | parent: "Core Abstraction" 763 | nav_order: 6 764 | --- 765 | 766 | # (Advanced) Parallel 767 | 768 | **Parallel** Nodes and Flows let you run multiple **Async** Nodes and Flows **concurrently**—for example, summarizing multiple texts at once. This can improve performance by overlapping I/O and compute. 769 | 770 | > Because of Python’s GIL, parallel nodes and flows can’t truly parallelize CPU-bound tasks (e.g., heavy numerical computations). However, they excel at overlapping I/O-bound work—like LLM calls, database queries, API requests, or file I/O. 771 | {: .warning } 772 | 773 | > - **Ensure Tasks Are Independent**: If each item depends on the output of a previous item, **do not** parallelize. 774 | > 775 | > - **Beware of Rate Limits**: Parallel calls can **quickly** trigger rate limits on LLM services. You may need a **throttling** mechanism (e.g., semaphores or sleep intervals). 776 | > 777 | > - **Consider Single-Node Batch APIs**: Some LLMs offer a **batch inference** API where you can send multiple prompts in a single call. This is more complex to implement but can be more efficient than launching many parallel requests and mitigates rate limits. 778 | {: .best-practice } 779 | 780 | ## AsyncParallelBatchNode 781 | 782 | Like **AsyncBatchNode**, but run `exec_async()` in **parallel**: 783 | 784 | ```python 785 | class ParallelSummaries(AsyncParallelBatchNode): 786 | async def prep_async(self, shared): 787 | # e.g., multiple texts 788 | return shared["texts"] 789 | 790 | async def exec_async(self, text): 791 | prompt = f"Summarize: {text}" 792 | return await call_llm_async(prompt) 793 | 794 | async def post_async(self, shared, prep_res, exec_res_list): 795 | shared["summary"] = "\n\n".join(exec_res_list) 796 | return "default" 797 | 798 | node = ParallelSummaries() 799 | flow = AsyncFlow(start=node) 800 | ``` 801 | 802 | ## AsyncParallelBatchFlow 803 | 804 | Parallel version of **BatchFlow**. Each iteration of the sub-flow runs **concurrently** using different parameters: 805 | 806 | ```python 807 | class SummarizeMultipleFiles(AsyncParallelBatchFlow): 808 | async def prep_async(self, shared): 809 | return [{"filename": f} for f in shared["files"]] 810 | 811 | sub_flow = AsyncFlow(start=LoadAndSummarizeFile()) 812 | parallel_flow = SummarizeMultipleFiles(start=sub_flow) 813 | await parallel_flow.run_async(shared) 814 | ``` 815 | 816 | ================================================ 817 | File: docs/design_pattern/agent.md 818 | ================================================ 819 | --- 820 | layout: default 821 | title: "Agent" 822 | parent: "Design Pattern" 823 | nav_order: 6 824 | --- 825 | 826 | # Agent 827 | 828 | Agent is a powerful design pattern, where node can take dynamic actions based on the context it receives. 829 | To express an agent, create a Node (the agent) with [branching](../core_abstraction/flow.md) to other nodes (Actions). 830 | 831 | > The core of build **performant** and **reliable** agents boils down to: 832 | > 833 | > 1. **Context Management:** Provide *clear, relevant context* so agents can understand the problem.E.g., Rather than dumping an entire chat history or entire files, use a [Workflow](./workflow.md) that filters out and includes only the most relevant information. 834 | > 835 | > 2. **Action Space:** Define *a well-structured, unambiguous, and easy-to-use* set of actions. For instance, avoid creating overlapping actions like `read_databases` and `read_csvs`. Instead, unify data sources (e.g., move CSVs into a database) and design a single action. The action can be parameterized (e.g., string for search) or programmable (e.g., SQL queries). 836 | {: .best-practice } 837 | 838 | ### Example: Search Agent 839 | 840 | This agent: 841 | 1. Decides whether to search or answer 842 | 2. If searches, loops back to decide if more search needed 843 | 3. Answers when enough context gathered 844 | 845 | ```python 846 | class DecideAction(Node): 847 | def prep(self, shared): 848 | context = shared.get("context", "No previous search") 849 | query = shared["query"] 850 | return query, context 851 | 852 | def exec(self, inputs): 853 | query, context = inputs 854 | prompt = f""" 855 | Given input: {query} 856 | Previous search results: {context} 857 | Should I: 1) Search web for more info 2) Answer with current knowledge 858 | Output in yaml: 859 | ```yaml 860 | action: search/answer 861 | reason: why this action 862 | search_term: search phrase if action is search 863 | ```""" 864 | resp = call_llm(prompt) 865 | yaml_str = resp.split("```yaml")[1].split("```")[0].strip() 866 | result = yaml.safe_load(yaml_str) 867 | 868 | assert isinstance(result, dict) 869 | assert "action" in result 870 | assert "reason" in result 871 | assert result["action"] in ["search", "answer"] 872 | if result["action"] == "search": 873 | assert "search_term" in result 874 | 875 | return result 876 | 877 | def post(self, shared, prep_res, exec_res): 878 | if exec_res["action"] == "search": 879 | shared["search_term"] = exec_res["search_term"] 880 | return exec_res["action"] 881 | 882 | class SearchWeb(Node): 883 | def prep(self, shared): 884 | return shared["search_term"] 885 | 886 | def exec(self, search_term): 887 | return search_web(search_term) 888 | 889 | def post(self, shared, prep_res, exec_res): 890 | prev_searches = shared.get("context", []) 891 | shared["context"] = prev_searches + [ 892 | {"term": shared["search_term"], "result": exec_res} 893 | ] 894 | return "decide" 895 | 896 | class DirectAnswer(Node): 897 | def prep(self, shared): 898 | return shared["query"], shared.get("context", "") 899 | 900 | def exec(self, inputs): 901 | query, context = inputs 902 | return call_llm(f"Context: {context}\nAnswer: {query}") 903 | 904 | def post(self, shared, prep_res, exec_res): 905 | print(f"Answer: {exec_res}") 906 | shared["answer"] = exec_res 907 | 908 | # Connect nodes 909 | decide = DecideAction() 910 | search = SearchWeb() 911 | answer = DirectAnswer() 912 | 913 | decide - "search" >> search 914 | decide - "answer" >> answer 915 | search - "decide" >> decide # Loop back 916 | 917 | flow = Flow(start=decide) 918 | flow.run({"query": "Who won the Nobel Prize in Physics 2024?"}) 919 | ``` 920 | 921 | ================================================ 922 | File: docs/design_pattern/mapreduce.md 923 | ================================================ 924 | --- 925 | layout: default 926 | title: "Map Reduce" 927 | parent: "Design Pattern" 928 | nav_order: 3 929 | --- 930 | 931 | # Map Reduce 932 | 933 | MapReduce is a design pattern suitable when you have either: 934 | - Large input data (e.g., multiple files to process), or 935 | - Large output data (e.g., multiple forms to fill) 936 | 937 | and there is a logical way to break the task into smaller, ideally independent parts. 938 | You first break down the task using [BatchNode](../core_abstraction/batch.md) in the map phase, followed by aggregation in the reduce phase. 939 | 940 | ### Example: Document Summarization 941 | 942 | ```python 943 | class MapSummaries(BatchNode): 944 | def prep(self, shared): return [shared["text"][i:i+10000] for i in range(0, len(shared["text"]), 10000)] 945 | def exec(self, chunk): return call_llm(f"Summarize this chunk: {chunk}") 946 | def post(self, shared, prep_res, exec_res_list): shared["summaries"] = exec_res_list 947 | 948 | class ReduceSummaries(Node): 949 | def prep(self, shared): return shared["summaries"] 950 | def exec(self, summaries): return call_llm(f"Combine these summaries: {summaries}") 951 | def post(self, shared, prep_res, exec_res): shared["final_summary"] = exec_res 952 | 953 | # Connect nodes 954 | map_node = MapSummaries() 955 | reduce_node = ReduceSummaries() 956 | map_node >> reduce_node 957 | 958 | # Create flow 959 | summarize_flow = Flow(start=map_node) 960 | summarize_flow.run(shared) 961 | ``` 962 | 963 | ================================================ 964 | File: docs/design_pattern/memory.md 965 | ================================================ 966 | --- 967 | layout: default 968 | title: "Chat Memory" 969 | parent: "Design Pattern" 970 | nav_order: 5 971 | --- 972 | 973 | # Chat Memory 974 | 975 | Multi-turn conversations require memory management to maintain context while avoiding overwhelming the LLM. 976 | 977 | ### 1. Naive Approach: Full History 978 | 979 | Sending the full chat history may overwhelm LLMs. 980 | 981 | ```python 982 | class ChatNode(Node): 983 | def prep(self, shared): 984 | if "history" not in shared: 985 | shared["history"] = [] 986 | user_input = input("You: ") 987 | return shared["history"], user_input 988 | 989 | def exec(self, inputs): 990 | history, user_input = inputs 991 | messages = [{"role": "system", "content": "You are a helpful assistant"}] 992 | for h in history: 993 | messages.append(h) 994 | messages.append({"role": "user", "content": user_input}) 995 | response = call_llm(messages) 996 | return response 997 | 998 | def post(self, shared, prep_res, exec_res): 999 | shared["history"].append({"role": "user", "content": prep_res[1]}) 1000 | shared["history"].append({"role": "assistant", "content": exec_res}) 1001 | return "continue" 1002 | 1003 | chat = ChatNode() 1004 | chat - "continue" >> chat 1005 | flow = Flow(start=chat) 1006 | ``` 1007 | 1008 | ### 2. Improved Memory Management 1009 | 1010 | We can: 1011 | 1. Limit the chat history to the most recent 4. 1012 | 2. Use [vector search](./tool.md) to retrieve relevant exchanges beyond the last 4. 1013 | 1014 | ```python 1015 | ################################ 1016 | # Node A: Retrieve user input & relevant messages 1017 | ################################ 1018 | class ChatRetrieve(Node): 1019 | def prep(self, s): 1020 | s.setdefault("history", []) 1021 | s.setdefault("memory_index", None) 1022 | user_input = input("You: ") 1023 | return user_input 1024 | 1025 | def exec(self, user_input): 1026 | emb = get_embedding(user_input) 1027 | relevant = [] 1028 | if len(shared["history"]) > 8 and shared["memory_index"]: 1029 | idx, _ = search_index(shared["memory_index"], emb, top_k=2) 1030 | relevant = [shared["history"][i[0]] for i in idx] 1031 | return (user_input, relevant) 1032 | 1033 | def post(self, s, p, r): 1034 | user_input, relevant = r 1035 | s["user_input"] = user_input 1036 | s["relevant"] = relevant 1037 | return "continue" 1038 | 1039 | ################################ 1040 | # Node B: Call LLM, update history + index 1041 | ################################ 1042 | class ChatReply(Node): 1043 | def prep(self, s): 1044 | user_input = s["user_input"] 1045 | recent = s["history"][-8:] 1046 | relevant = s.get("relevant", []) 1047 | return user_input, recent, relevant 1048 | 1049 | def exec(self, inputs): 1050 | user_input, recent, relevant = inputs 1051 | msgs = [{"role":"system","content":"You are a helpful assistant."}] 1052 | if relevant: 1053 | msgs.append({"role":"system","content":f"Relevant: {relevant}"}) 1054 | msgs.extend(recent) 1055 | msgs.append({"role":"user","content":user_input}) 1056 | ans = call_llm(msgs) 1057 | return ans 1058 | 1059 | def post(self, s, pre, ans): 1060 | user_input, _, _ = pre 1061 | s["history"].append({"role":"user","content":user_input}) 1062 | s["history"].append({"role":"assistant","content":ans}) 1063 | 1064 | # Manage memory index 1065 | if len(s["history"]) == 8: 1066 | embs = [] 1067 | for i in range(0, 8, 2): 1068 | text = s["history"][i]["content"] + " " + s["history"][i+1]["content"] 1069 | embs.append(get_embedding(text)) 1070 | s["memory_index"] = create_index(embs) 1071 | elif len(s["history"]) > 8: 1072 | text = s["history"][-2]["content"] + " " + s["history"][-1]["content"] 1073 | new_emb = np.array([get_embedding(text)]).astype('float32') 1074 | s["memory_index"].add(new_emb) 1075 | 1076 | print(f"Assistant: {ans}") 1077 | return "continue" 1078 | 1079 | ################################ 1080 | # Flow wiring 1081 | ################################ 1082 | retrieve = ChatRetrieve() 1083 | reply = ChatReply() 1084 | retrieve - "continue" >> reply 1085 | reply - "continue" >> retrieve 1086 | 1087 | flow = Flow(start=retrieve) 1088 | shared = {} 1089 | flow.run(shared) 1090 | ``` 1091 | 1092 | ================================================ 1093 | File: docs/design_pattern/multi_agent.md 1094 | ================================================ 1095 | --- 1096 | layout: default 1097 | title: "(Advanced) Multi-Agents" 1098 | parent: "Design Pattern" 1099 | nav_order: 7 1100 | --- 1101 | 1102 | # (Advanced) Multi-Agents 1103 | 1104 | Multiple [Agents](./flow.md) can work together by handling subtasks and communicating the progress. 1105 | Communication between agents is typically implemented using message queues in shared storage. 1106 | 1107 | > Most of time, you don't need Multi-Agents. Start with a simple solution first. 1108 | {: .best-practice } 1109 | 1110 | ### Example Agent Communication: Message Queue 1111 | 1112 | Here's a simple example showing how to implement agent communication using `asyncio.Queue`. 1113 | The agent listens for messages, processes them, and continues listening: 1114 | 1115 | ```python 1116 | class AgentNode(AsyncNode): 1117 | async def prep_async(self, _): 1118 | message_queue = self.params["messages"] 1119 | message = await message_queue.get() 1120 | print(f"Agent received: {message}") 1121 | return message 1122 | 1123 | # Create node and flow 1124 | agent = AgentNode() 1125 | agent >> agent # connect to self 1126 | flow = AsyncFlow(start=agent) 1127 | 1128 | # Create heartbeat sender 1129 | async def send_system_messages(message_queue): 1130 | counter = 0 1131 | messages = [ 1132 | "System status: all systems operational", 1133 | "Memory usage: normal", 1134 | "Network connectivity: stable", 1135 | "Processing load: optimal" 1136 | ] 1137 | 1138 | while True: 1139 | message = f"{messages[counter % len(messages)]} | timestamp_{counter}" 1140 | await message_queue.put(message) 1141 | counter += 1 1142 | await asyncio.sleep(1) 1143 | 1144 | async def main(): 1145 | message_queue = asyncio.Queue() 1146 | shared = {} 1147 | flow.set_params({"messages": message_queue}) 1148 | 1149 | # Run both coroutines 1150 | await asyncio.gather( 1151 | flow.run_async(shared), 1152 | send_system_messages(message_queue) 1153 | ) 1154 | 1155 | asyncio.run(main()) 1156 | ``` 1157 | 1158 | The output: 1159 | 1160 | ``` 1161 | Agent received: System status: all systems operational | timestamp_0 1162 | Agent received: Memory usage: normal | timestamp_1 1163 | Agent received: Network connectivity: stable | timestamp_2 1164 | Agent received: Processing load: optimal | timestamp_3 1165 | ``` 1166 | 1167 | ### Interactive Multi-Agent Example: Taboo Game 1168 | 1169 | Here's a more complex example where two agents play the word-guessing game Taboo. 1170 | One agent provides hints while avoiding forbidden words, and another agent tries to guess the target word: 1171 | 1172 | ```python 1173 | class AsyncHinter(AsyncNode): 1174 | async def prep_async(self, shared): 1175 | guess = await shared["hinter_queue"].get() 1176 | if guess == "GAME_OVER": 1177 | return None 1178 | return shared["target_word"], shared["forbidden_words"], shared.get("past_guesses", []) 1179 | 1180 | async def exec_async(self, inputs): 1181 | if inputs is None: 1182 | return None 1183 | target, forbidden, past_guesses = inputs 1184 | prompt = f"Generate hint for '{target}'\nForbidden words: {forbidden}" 1185 | if past_guesses: 1186 | prompt += f"\nPrevious wrong guesses: {past_guesses}\nMake hint more specific." 1187 | prompt += "\nUse at most 5 words." 1188 | 1189 | hint = call_llm(prompt) 1190 | print(f"\nHinter: Here's your hint - {hint}") 1191 | return hint 1192 | 1193 | async def post_async(self, shared, prep_res, exec_res): 1194 | if exec_res is None: 1195 | return "end" 1196 | await shared["guesser_queue"].put(exec_res) 1197 | return "continue" 1198 | 1199 | class AsyncGuesser(AsyncNode): 1200 | async def prep_async(self, shared): 1201 | hint = await shared["guesser_queue"].get() 1202 | return hint, shared.get("past_guesses", []) 1203 | 1204 | async def exec_async(self, inputs): 1205 | hint, past_guesses = inputs 1206 | prompt = f"Given hint: {hint}, past wrong guesses: {past_guesses}, make a new guess. Directly reply a single word:" 1207 | guess = call_llm(prompt) 1208 | print(f"Guesser: I guess it's - {guess}") 1209 | return guess 1210 | 1211 | async def post_async(self, shared, prep_res, exec_res): 1212 | if exec_res.lower() == shared["target_word"].lower(): 1213 | print("Game Over - Correct guess!") 1214 | await shared["hinter_queue"].put("GAME_OVER") 1215 | return "end" 1216 | 1217 | if "past_guesses" not in shared: 1218 | shared["past_guesses"] = [] 1219 | shared["past_guesses"].append(exec_res) 1220 | 1221 | await shared["hinter_queue"].put(exec_res) 1222 | return "continue" 1223 | 1224 | async def main(): 1225 | # Set up game 1226 | shared = { 1227 | "target_word": "nostalgia", 1228 | "forbidden_words": ["memory", "past", "remember", "feeling", "longing"], 1229 | "hinter_queue": asyncio.Queue(), 1230 | "guesser_queue": asyncio.Queue() 1231 | } 1232 | 1233 | print("Game starting!") 1234 | print(f"Target word: {shared['target_word']}") 1235 | print(f"Forbidden words: {shared['forbidden_words']}") 1236 | 1237 | # Initialize by sending empty guess to hinter 1238 | await shared["hinter_queue"].put("") 1239 | 1240 | # Create nodes and flows 1241 | hinter = AsyncHinter() 1242 | guesser = AsyncGuesser() 1243 | 1244 | # Set up flows 1245 | hinter_flow = AsyncFlow(start=hinter) 1246 | guesser_flow = AsyncFlow(start=guesser) 1247 | 1248 | # Connect nodes to themselves 1249 | hinter - "continue" >> hinter 1250 | guesser - "continue" >> guesser 1251 | 1252 | # Run both agents concurrently 1253 | await asyncio.gather( 1254 | hinter_flow.run_async(shared), 1255 | guesser_flow.run_async(shared) 1256 | ) 1257 | 1258 | asyncio.run(main()) 1259 | ``` 1260 | 1261 | The Output: 1262 | 1263 | ``` 1264 | Game starting! 1265 | Target word: nostalgia 1266 | Forbidden words: ['memory', 'past', 'remember', 'feeling', 'longing'] 1267 | 1268 | Hinter: Here's your hint - Thinking of childhood summer days 1269 | Guesser: I guess it's - popsicle 1270 | 1271 | Hinter: Here's your hint - When childhood cartoons make you emotional 1272 | Guesser: I guess it's - nostalgic 1273 | 1274 | Hinter: Here's your hint - When old songs move you 1275 | Guesser: I guess it's - memories 1276 | 1277 | Hinter: Here's your hint - That warm emotion about childhood 1278 | Guesser: I guess it's - nostalgia 1279 | Game Over - Correct guess! 1280 | ``` 1281 | 1282 | ================================================ 1283 | File: docs/design_pattern/rag.md 1284 | ================================================ 1285 | --- 1286 | layout: default 1287 | title: "RAG" 1288 | parent: "Design Pattern" 1289 | nav_order: 4 1290 | --- 1291 | 1292 | # RAG (Retrieval Augmented Generation) 1293 | 1294 | For certain LLM tasks like answering questions, providing context is essential. 1295 | Use [vector search](../utility_function/tool.md) to find relevant context for LLM responses. 1296 | 1297 | ### Example: Question Answering 1298 | 1299 | ```python 1300 | class PrepareEmbeddings(Node): 1301 | def prep(self, shared): 1302 | return shared["texts"] 1303 | 1304 | def exec(self, texts): 1305 | # Embed each text chunk 1306 | embs = [get_embedding(t) for t in texts] 1307 | return embs 1308 | 1309 | def post(self, shared, prep_res, exec_res): 1310 | shared["search_index"] = create_index(exec_res) 1311 | # no action string means "default" 1312 | 1313 | class AnswerQuestion(Node): 1314 | def prep(self, shared): 1315 | question = input("Enter question: ") 1316 | return question 1317 | 1318 | def exec(self, question): 1319 | q_emb = get_embedding(question) 1320 | idx, _ = search_index(shared["search_index"], q_emb, top_k=1) 1321 | best_id = idx[0][0] 1322 | relevant_text = shared["texts"][best_id] 1323 | prompt = f"Question: {question}\nContext: {relevant_text}\nAnswer:" 1324 | return call_llm(prompt) 1325 | 1326 | def post(self, shared, p, answer): 1327 | print("Answer:", answer) 1328 | 1329 | ############################################ 1330 | # Wire up the flow 1331 | prep = PrepareEmbeddings() 1332 | qa = AnswerQuestion() 1333 | prep >> qa 1334 | 1335 | flow = Flow(start=prep) 1336 | 1337 | # Example usage 1338 | shared = {"texts": ["I love apples", "Cats are great", "The sky is blue"]} 1339 | flow.run(shared) 1340 | ``` 1341 | 1342 | ================================================ 1343 | File: docs/design_pattern/structure.md 1344 | ================================================ 1345 | --- 1346 | layout: default 1347 | title: "Structured Output" 1348 | parent: "Design Pattern" 1349 | nav_order: 1 1350 | --- 1351 | 1352 | # Structured Output 1353 | 1354 | In many use cases, you may want the LLM to output a specific structure, such as a list or a dictionary with predefined keys. 1355 | 1356 | There are several approaches to achieve a structured output: 1357 | - **Prompting** the LLM to strictly return a defined structure. 1358 | - Using LLMs that natively support **schema enforcement**. 1359 | - **Post-processing** the LLM's response to extract structured content. 1360 | 1361 | In practice, **Prompting** is simple and reliable for modern LLMs. 1362 | 1363 | ### Example Use Cases 1364 | 1365 | - Extracting Key Information 1366 | 1367 | ```yaml 1368 | product: 1369 | name: Widget Pro 1370 | price: 199.99 1371 | description: | 1372 | A high-quality widget designed for professionals. 1373 | Recommended for advanced users. 1374 | ``` 1375 | 1376 | - Summarizing Documents into Bullet Points 1377 | 1378 | ```yaml 1379 | summary: 1380 | - This product is easy to use. 1381 | - It is cost-effective. 1382 | - Suitable for all skill levels. 1383 | ``` 1384 | 1385 | - Generating Configuration Files 1386 | 1387 | ```yaml 1388 | server: 1389 | host: 127.0.0.1 1390 | port: 8080 1391 | ssl: true 1392 | ``` 1393 | 1394 | ## Prompt Engineering 1395 | 1396 | When prompting the LLM to produce **structured** output: 1397 | 1. **Wrap** the structure in code fences (e.g., `yaml`). 1398 | 2. **Validate** that all required fields exist (and let `Node` handles retry). 1399 | 1400 | ### Example Text Summarization 1401 | 1402 | ```python 1403 | class SummarizeNode(Node): 1404 | def exec(self, prep_res): 1405 | # Suppose `prep_res` is the text to summarize. 1406 | prompt = f""" 1407 | Please summarize the following text as YAML, with exactly 3 bullet points 1408 | 1409 | {prep_res} 1410 | 1411 | Now, output: 1412 | ```yaml 1413 | summary: 1414 | - bullet 1 1415 | - bullet 2 1416 | - bullet 3 1417 | ```""" 1418 | response = call_llm(prompt) 1419 | yaml_str = response.split("```yaml")[1].split("```")[0].strip() 1420 | 1421 | import yaml 1422 | structured_result = yaml.safe_load(yaml_str) 1423 | 1424 | assert "summary" in structured_result 1425 | assert isinstance(structured_result["summary"], list) 1426 | 1427 | return structured_result 1428 | ``` 1429 | 1430 | > Besides using `assert` statements, another popular way to validate schemas is [Pydantic](https://github.com/pydantic/pydantic) 1431 | {: .note } 1432 | 1433 | ### Why YAML instead of JSON? 1434 | 1435 | Current LLMs struggle with escaping. YAML is easier with strings since they don't always need quotes. 1436 | 1437 | **In JSON** 1438 | 1439 | ```json 1440 | { 1441 | "dialogue": "Alice said: \"Hello Bob.\\nHow are you?\\nI am good.\"" 1442 | } 1443 | ``` 1444 | 1445 | - Every double quote inside the string must be escaped with `\"`. 1446 | - Each newline in the dialogue must be represented as `\n`. 1447 | 1448 | **In YAML** 1449 | 1450 | ```yaml 1451 | dialogue: | 1452 | Alice said: "Hello Bob. 1453 | How are you? 1454 | I am good." 1455 | ``` 1456 | 1457 | - No need to escape interior quotes—just place the entire text under a block literal (`|`). 1458 | - Newlines are naturally preserved without needing `\n`. 1459 | 1460 | ================================================ 1461 | File: docs/design_pattern/workflow.md 1462 | ================================================ 1463 | --- 1464 | layout: default 1465 | title: "Workflow" 1466 | parent: "Design Pattern" 1467 | nav_order: 2 1468 | --- 1469 | 1470 | # Workflow 1471 | 1472 | Many real-world tasks are too complex for one LLM call. The solution is to decompose them into a [chain](../core_abstraction/flow.md) of multiple Nodes. 1473 | 1474 | > - You don't want to make each task **too coarse**, because it may be *too complex for one LLM call*. 1475 | > - You don't want to make each task **too granular**, because then *the LLM call doesn't have enough context* and results are *not consistent across nodes*. 1476 | > 1477 | > You usually need multiple *iterations* to find the *sweet spot*. If the task has too many *edge cases*, consider using [Agents](./agent.md). 1478 | {: .best-practice } 1479 | 1480 | ### Example: Article Writing 1481 | 1482 | ```python 1483 | class GenerateOutline(Node): 1484 | def prep(self, shared): return shared["topic"] 1485 | def exec(self, topic): return call_llm(f"Create a detailed outline for an article about {topic}") 1486 | def post(self, shared, prep_res, exec_res): shared["outline"] = exec_res 1487 | 1488 | class WriteSection(Node): 1489 | def prep(self, shared): return shared["outline"] 1490 | def exec(self, outline): return call_llm(f"Write content based on this outline: {outline}") 1491 | def post(self, shared, prep_res, exec_res): shared["draft"] = exec_res 1492 | 1493 | class ReviewAndRefine(Node): 1494 | def prep(self, shared): return shared["draft"] 1495 | def exec(self, draft): return call_llm(f"Review and improve this draft: {draft}") 1496 | def post(self, shared, prep_res, exec_res): shared["final_article"] = exec_res 1497 | 1498 | # Connect nodes 1499 | outline = GenerateOutline() 1500 | write = WriteSection() 1501 | review = ReviewAndRefine() 1502 | 1503 | outline >> write >> review 1504 | 1505 | # Create and run flow 1506 | writing_flow = Flow(start=outline) 1507 | shared = {"topic": "AI Safety"} 1508 | writing_flow.run(shared) 1509 | ``` 1510 | 1511 | For *dynamic cases*, consider using [Agents](./agent.md). 1512 | 1513 | ================================================ 1514 | File: docs/utility_function/llm.md 1515 | ================================================ 1516 | --- 1517 | layout: default 1518 | title: "LLM Wrapper" 1519 | parent: "Utility Function" 1520 | nav_order: 1 1521 | --- 1522 | 1523 | # LLM Wrappers 1524 | 1525 | We **don't** provide built-in LLM wrappers. Instead, please implement your own, for example by asking an assistant like ChatGPT or Claude. If you ask ChatGPT to "implement a `call_llm` function that takes a prompt and returns the LLM response," you shall get something like: 1526 | 1527 | ```python 1528 | def call_llm(prompt): 1529 | from openai import OpenAI 1530 | client = OpenAI(api_key="YOUR_API_KEY_HERE") 1531 | r = client.chat.completions.create( 1532 | model="gpt-4o", 1533 | messages=[{"role": "user", "content": prompt}] 1534 | ) 1535 | return r.choices[0].message.content 1536 | 1537 | # Example usage 1538 | call_llm("How are you?") 1539 | ``` 1540 | 1541 | > Store the API key in an environment variable like OPENAI_API_KEY for security. 1542 | {: .note } 1543 | 1544 | ## Improvements 1545 | Feel free to enhance your `call_llm` function as needed. Here are examples: 1546 | 1547 | - Handle chat history: 1548 | 1549 | ```python 1550 | def call_llm(messages): 1551 | from openai import OpenAI 1552 | client = OpenAI(api_key="YOUR_API_KEY_HERE") 1553 | r = client.chat.completions.create( 1554 | model="gpt-4o", 1555 | messages=messages 1556 | ) 1557 | return r.choices[0].message.content 1558 | ``` 1559 | 1560 | - Add in-memory caching 1561 | 1562 | ```python 1563 | from functools import lru_cache 1564 | 1565 | @lru_cache(maxsize=1000) 1566 | def call_llm(prompt): 1567 | # Your implementation here 1568 | pass 1569 | ``` 1570 | 1571 | > ⚠️ Caching conflicts with Node retries, as retries yield the same result. 1572 | > 1573 | > To address this, you could use cached results only if not retried. 1574 | {: .warning } 1575 | 1576 | 1577 | ```python 1578 | from functools import lru_cache 1579 | 1580 | @lru_cache(maxsize=1000) 1581 | def cached_call(prompt): 1582 | pass 1583 | 1584 | def call_llm(prompt, use_cache): 1585 | if use_cache: 1586 | return cached_call(prompt) 1587 | # Call the underlying function directly 1588 | return cached_call.__wrapped__(prompt) 1589 | 1590 | class SummarizeNode(Node): 1591 | def exec(self, text): 1592 | return call_llm(f"Summarize: {text}", self.cur_retry==0) 1593 | ``` 1594 | 1595 | - Enable logging: 1596 | 1597 | ```python 1598 | def call_llm(prompt): 1599 | import logging 1600 | logging.info(f"Prompt: {prompt}") 1601 | response = ... # Your implementation here 1602 | logging.info(f"Response: {response}") 1603 | return response 1604 | ``` 1605 | 1606 | ## Why Not Provide Built-in LLM Wrappers? 1607 | I believe it is a **bad practice** to provide LLM-specific implementations in a general framework: 1608 | - **LLM APIs change frequently**. Hardcoding them makes maintenance a nightmare. 1609 | - You may need **flexibility** to switch vendors, use fine-tuned models, or deploy local LLMs. 1610 | - You may need **optimizations** like prompt caching, request batching, or response streaming. 1611 | 1612 | ================================================ 1613 | File: docs/utility_function/tool.md 1614 | ================================================ 1615 | --- 1616 | layout: default 1617 | title: "Tool" 1618 | parent: "Utility Function" 1619 | nav_order: 2 1620 | --- 1621 | 1622 | # Tool 1623 | 1624 | Similar to LLM wrappers, we **don't** provide built-in tools. Here, we recommend some *minimal* (and incomplete) implementations of commonly used tools. These examples can serve as a starting point for your own tooling. 1625 | 1626 | --- 1627 | 1628 | ## 1. Embedding Calls 1629 | 1630 | ```python 1631 | def get_embedding(text): 1632 | from openai import OpenAI 1633 | client = OpenAI(api_key="YOUR_API_KEY_HERE") 1634 | r = client.embeddings.create( 1635 | model="text-embedding-ada-002", 1636 | input=text 1637 | ) 1638 | return r.data[0].embedding 1639 | 1640 | get_embedding("What's the meaning of life?") 1641 | ``` 1642 | 1643 | --- 1644 | 1645 | ## 2. Vector Database (Faiss) 1646 | 1647 | ```python 1648 | import faiss 1649 | import numpy as np 1650 | 1651 | def create_index(embeddings): 1652 | dim = len(embeddings[0]) 1653 | index = faiss.IndexFlatL2(dim) 1654 | index.add(np.array(embeddings).astype('float32')) 1655 | return index 1656 | 1657 | def search_index(index, query_embedding, top_k=5): 1658 | D, I = index.search( 1659 | np.array([query_embedding]).astype('float32'), 1660 | top_k 1661 | ) 1662 | return I, D 1663 | 1664 | index = create_index(embeddings) 1665 | search_index(index, query_embedding) 1666 | ``` 1667 | 1668 | --- 1669 | 1670 | ## 3. Local Database 1671 | 1672 | ```python 1673 | import sqlite3 1674 | 1675 | def execute_sql(query): 1676 | conn = sqlite3.connect("mydb.db") 1677 | cursor = conn.cursor() 1678 | cursor.execute(query) 1679 | result = cursor.fetchall() 1680 | conn.commit() 1681 | conn.close() 1682 | return result 1683 | ``` 1684 | 1685 | > ⚠️ Beware of SQL injection risk 1686 | {: .warning } 1687 | 1688 | --- 1689 | 1690 | ## 4. Python Function Execution 1691 | 1692 | ```python 1693 | def run_code(code_str): 1694 | env = {} 1695 | exec(code_str, env) 1696 | return env 1697 | 1698 | run_code("print('Hello, world!')") 1699 | ``` 1700 | 1701 | > ⚠️ exec() is dangerous with untrusted input 1702 | {: .warning } 1703 | 1704 | 1705 | --- 1706 | 1707 | ## 5. PDF Extraction 1708 | 1709 | If your PDFs are text-based, use PyMuPDF: 1710 | 1711 | ```python 1712 | import fitz # PyMuPDF 1713 | 1714 | def extract_text(pdf_path): 1715 | doc = fitz.open(pdf_path) 1716 | text = "" 1717 | for page in doc: 1718 | text += page.get_text() 1719 | doc.close() 1720 | return text 1721 | 1722 | extract_text("document.pdf") 1723 | ``` 1724 | 1725 | For image-based PDFs (e.g., scanned), OCR is needed. A easy and fast option is using an LLM with vision capabilities: 1726 | 1727 | ```python 1728 | from openai import OpenAI 1729 | import base64 1730 | 1731 | def call_llm_vision(prompt, image_data): 1732 | client = OpenAI(api_key="YOUR_API_KEY_HERE") 1733 | img_base64 = base64.b64encode(image_data).decode('utf-8') 1734 | 1735 | response = client.chat.completions.create( 1736 | model="gpt-4o", 1737 | messages=[{ 1738 | "role": "user", 1739 | "content": [ 1740 | {"type": "text", "text": prompt}, 1741 | {"type": "image_url", 1742 | "image_url": {"url": f"data:image/png;base64,{img_base64}"}} 1743 | ] 1744 | }] 1745 | ) 1746 | 1747 | return response.choices[0].message.content 1748 | 1749 | pdf_document = fitz.open("document.pdf") 1750 | page_num = 0 1751 | page = pdf_document[page_num] 1752 | pix = page.get_pixmap() 1753 | img_data = pix.tobytes("png") 1754 | 1755 | call_llm_vision("Extract text from this image", img_data) 1756 | ``` 1757 | 1758 | --- 1759 | 1760 | ## 6. Web Crawling 1761 | 1762 | ```python 1763 | def crawl_web(url): 1764 | import requests 1765 | from bs4 import BeautifulSoup 1766 | html = requests.get(url).text 1767 | soup = BeautifulSoup(html, "html.parser") 1768 | return soup.title.string, soup.get_text() 1769 | ``` 1770 | 1771 | --- 1772 | 1773 | ## 7. Basic Search (SerpAPI example) 1774 | 1775 | ```python 1776 | def search_google(query): 1777 | import requests 1778 | params = { 1779 | "engine": "google", 1780 | "q": query, 1781 | "api_key": "YOUR_API_KEY" 1782 | } 1783 | r = requests.get("https://serpapi.com/search", params=params) 1784 | return r.json() 1785 | ``` 1786 | 1787 | --- 1788 | 1789 | 1790 | ## 8. Audio Transcription (OpenAI Whisper) 1791 | 1792 | ```python 1793 | def transcribe_audio(file_path): 1794 | import openai 1795 | audio_file = open(file_path, "rb") 1796 | transcript = openai.Audio.transcribe("whisper-1", audio_file) 1797 | return transcript["text"] 1798 | ``` 1799 | 1800 | --- 1801 | 1802 | ## 9. Text-to-Speech (TTS) 1803 | 1804 | ```python 1805 | def text_to_speech(text): 1806 | import pyttsx3 1807 | engine = pyttsx3.init() 1808 | engine.say(text) 1809 | engine.runAndWait() 1810 | ``` 1811 | 1812 | --- 1813 | 1814 | ## 10. Sending Email 1815 | 1816 | ```python 1817 | def send_email(to_address, subject, body, from_address, password): 1818 | import smtplib 1819 | from email.mime.text import MIMEText 1820 | 1821 | msg = MIMEText(body) 1822 | msg["Subject"] = subject 1823 | msg["From"] = from_address 1824 | msg["To"] = to_address 1825 | 1826 | with smtplib.SMTP_SSL("smtp.gmail.com", 465) as server: 1827 | server.login(from_address, password) 1828 | server.sendmail(from_address, [to_address], msg.as_string()) 1829 | ``` 1830 | 1831 | ================================================ 1832 | File: docs/utility_function/viz.md 1833 | ================================================ 1834 | --- 1835 | layout: default 1836 | title: "Viz and Debug" 1837 | parent: "Utility Function" 1838 | nav_order: 3 1839 | --- 1840 | 1841 | # Visualization and Debugging 1842 | 1843 | Similar to LLM wrappers, we **don't** provide built-in visualization and debugging. Here, we recommend some *minimal* (and incomplete) implementations These examples can serve as a starting point for your own tooling. 1844 | 1845 | ## 1. Visualization with Mermaid 1846 | 1847 | This code recursively traverses the nested graph, assigns unique IDs to each node, and treats Flow nodes as subgraphs to generate Mermaid syntax for a hierarchical visualization. 1848 | 1849 | {% raw %} 1850 | ```python 1851 | def build_mermaid(start): 1852 | ids, visited, lines = {}, set(), ["graph LR"] 1853 | ctr = 1 1854 | def get_id(n): 1855 | nonlocal ctr 1856 | return ids[n] if n in ids else (ids.setdefault(n, f"N{ctr}"), (ctr := ctr + 1))[0] 1857 | def link(a, b): 1858 | lines.append(f" {a} --> {b}") 1859 | def walk(node, parent=None): 1860 | if node in visited: 1861 | return parent and link(parent, get_id(node)) 1862 | visited.add(node) 1863 | if isinstance(node, Flow): 1864 | node.start and parent and link(parent, get_id(node.start)) 1865 | lines.append(f"\n subgraph sub_flow_{get_id(node)}[{type(node).__name__}]") 1866 | node.start and walk(node.start) 1867 | for nxt in node.successors.values(): 1868 | node.start and walk(nxt, get_id(node.start)) or (parent and link(parent, get_id(nxt))) or walk(nxt) 1869 | lines.append(" end\n") 1870 | else: 1871 | lines.append(f" {(nid := get_id(node))}['{type(node).__name__}']") 1872 | parent and link(parent, nid) 1873 | [walk(nxt, nid) for nxt in node.successors.values()] 1874 | walk(start) 1875 | return "\n".join(lines) 1876 | ``` 1877 | {% endraw %} 1878 | 1879 | 1880 | For example, suppose we have a complex Flow for data science: 1881 | 1882 | ```python 1883 | class DataPrepBatchNode(BatchNode): 1884 | def prep(self,shared): return [] 1885 | class ValidateDataNode(Node): pass 1886 | class FeatureExtractionNode(Node): pass 1887 | class TrainModelNode(Node): pass 1888 | class EvaluateModelNode(Node): pass 1889 | class ModelFlow(Flow): pass 1890 | class DataScienceFlow(Flow):pass 1891 | 1892 | feature_node = FeatureExtractionNode() 1893 | train_node = TrainModelNode() 1894 | evaluate_node = EvaluateModelNode() 1895 | feature_node >> train_node >> evaluate_node 1896 | model_flow = ModelFlow(start=feature_node) 1897 | data_prep_node = DataPrepBatchNode() 1898 | validate_node = ValidateDataNode() 1899 | data_prep_node >> validate_node >> model_flow 1900 | data_science_flow = DataScienceFlow(start=data_prep_node) 1901 | result = build_mermaid(start=data_science_flow) 1902 | ``` 1903 | 1904 | The code generates a Mermaid diagram: 1905 | 1906 | ```mermaid 1907 | graph LR 1908 | subgraph sub_flow_N1[DataScienceFlow] 1909 | N2['DataPrepBatchNode'] 1910 | N3['ValidateDataNode'] 1911 | N2 --> N3 1912 | N3 --> N4 1913 | 1914 | subgraph sub_flow_N5[ModelFlow] 1915 | N4['FeatureExtractionNode'] 1916 | N6['TrainModelNode'] 1917 | N4 --> N6 1918 | N7['EvaluateModelNode'] 1919 | N6 --> N7 1920 | end 1921 | 1922 | end 1923 | ``` 1924 | 1925 | ## 2. Call Stack Debugging 1926 | 1927 | It would be useful to print the Node call stacks for debugging. This can be achieved by inspecting the runtime call stack: 1928 | 1929 | ```python 1930 | import inspect 1931 | 1932 | def get_node_call_stack(): 1933 | stack = inspect.stack() 1934 | node_names = [] 1935 | seen_ids = set() 1936 | for frame_info in stack[1:]: 1937 | local_vars = frame_info.frame.f_locals 1938 | if 'self' in local_vars: 1939 | caller_self = local_vars['self'] 1940 | if isinstance(caller_self, BaseNode) and id(caller_self) not in seen_ids: 1941 | seen_ids.add(id(caller_self)) 1942 | node_names.append(type(caller_self).__name__) 1943 | return node_names 1944 | ``` 1945 | 1946 | For example, suppose we have a complex Flow for data science: 1947 | 1948 | ```python 1949 | class DataPrepBatchNode(BatchNode): 1950 | def prep(self, shared): return [] 1951 | class ValidateDataNode(Node): pass 1952 | class FeatureExtractionNode(Node): pass 1953 | class TrainModelNode(Node): pass 1954 | class EvaluateModelNode(Node): 1955 | def prep(self, shared): 1956 | stack = get_node_call_stack() 1957 | print("Call stack:", stack) 1958 | class ModelFlow(Flow): pass 1959 | class DataScienceFlow(Flow):pass 1960 | 1961 | feature_node = FeatureExtractionNode() 1962 | train_node = TrainModelNode() 1963 | evaluate_node = EvaluateModelNode() 1964 | feature_node >> train_node >> evaluate_node 1965 | model_flow = ModelFlow(start=feature_node) 1966 | data_prep_node = DataPrepBatchNode() 1967 | validate_node = ValidateDataNode() 1968 | data_prep_node >> validate_node >> model_flow 1969 | data_science_flow = DataScienceFlow(start=data_prep_node) 1970 | data_science_flow.run({}) 1971 | ``` 1972 | 1973 | The output would be: `Call stack: ['EvaluateModelNode', 'ModelFlow', 'DataScienceFlow']` --------------------------------------------------------------------------------