├── .clinerules ├── .cursorrules ├── .dockerignore ├── .env.sample ├── .gitignore ├── .windsurfrules ├── Dockerfile ├── LICENSE ├── README.md ├── assets ├── banner.png ├── example.png └── youtube_thumbnail.png ├── docs ├── AutoGen Core │ ├── 01_agent.md │ ├── 02_messaging_system__topic___subscription_.md │ ├── 03_agentruntime.md │ ├── 04_tool.md │ ├── 05_chatcompletionclient.md │ ├── 06_chatcompletioncontext.md │ ├── 07_memory.md │ ├── 08_component.md │ └── index.md ├── Browser Use │ ├── 01_agent.md │ ├── 02_system_prompt.md │ ├── 03_browsercontext.md │ ├── 04_dom_representation.md │ ├── 05_action_controller___registry.md │ ├── 06_message_manager.md │ ├── 07_data_structures__views_.md │ ├── 08_telemetry_service.md │ └── index.md ├── Celery │ ├── 01_celery_app.md │ ├── 02_configuration.md │ ├── 03_task.md │ ├── 04_broker_connection__amqp_.md │ ├── 05_worker.md │ ├── 06_result_backend.md │ ├── 07_beat__scheduler_.md │ ├── 08_canvas__signatures___primitives_.md │ ├── 09_events.md │ ├── 10_bootsteps.md │ └── index.md ├── Click │ ├── 01_command___group.md │ ├── 02_decorators.md │ ├── 03_parameter__option___argument_.md │ ├── 04_paramtype.md │ ├── 05_context.md │ ├── 06_term_ui__terminal_user_interface_.md │ ├── 07_click_exceptions.md │ └── index.md ├── Codex │ ├── 01_terminal_ui__ink_components_.md │ ├── 02_input_handling__textbuffer_editor_.md │ ├── 03_agent_loop.md │ ├── 04_approval_policy___security.md │ ├── 05_response___tool_call_handling.md │ ├── 06_command_execution___sandboxing.md │ ├── 07_configuration_management.md │ ├── 08_single_pass_mode.md │ └── index.md ├── Crawl4AI │ ├── 01_asynccrawlerstrategy.md │ ├── 02_asyncwebcrawler.md │ ├── 03_crawlerrunconfig.md │ ├── 04_contentscrapingstrategy.md │ ├── 05_relevantcontentfilter.md │ ├── 06_extractionstrategy.md │ ├── 07_crawlresult.md │ ├── 08_deepcrawlstrategy.md │ ├── 09_cachecontext___cachemode.md │ ├── 10_basedispatcher.md │ └── index.md ├── CrewAI │ ├── 01_crew.md │ ├── 02_agent.md │ ├── 03_task.md │ ├── 04_tool.md │ ├── 05_process.md │ ├── 06_llm.md │ ├── 07_memory.md │ ├── 08_knowledge.md │ └── index.md ├── DSPy │ ├── 01_module___program.md │ ├── 02_signature.md │ ├── 03_example.md │ ├── 04_predict.md │ ├── 05_lm__language_model_client_.md │ ├── 06_rm__retrieval_model_client_.md │ ├── 07_evaluate.md │ ├── 08_teleprompter___optimizer.md │ ├── 09_adapter.md │ ├── 10_settings.md │ └── index.md ├── FastAPI │ ├── 01_fastapi_application___routing.md │ ├── 02_path_operations___parameter_declaration.md │ ├── 03_data_validation___serialization__pydantic_.md │ ├── 04_openapi___automatic_docs.md │ ├── 05_dependency_injection.md │ ├── 06_error_handling.md │ ├── 07_security_utilities.md │ ├── 08_background_tasks.md │ └── index.md ├── Flask │ ├── 01_application_object___flask__.md │ ├── 02_routing_system.md │ ├── 03_request_and_response_objects.md │ ├── 04_templating__jinja2_integration_.md │ ├── 05_context_globals___current_app____request____session____g__.md │ ├── 06_configuration___config__.md │ ├── 07_application_and_request_contexts.md │ ├── 08_blueprints.md │ └── index.md ├── Google A2A │ ├── 01_agent_card.md │ ├── 02_task.md │ ├── 03_a2a_protocol___core_types.md │ ├── 04_a2a_server_implementation.md │ ├── 05_a2a_client_implementation.md │ ├── 06_task_handling_logic__server_side_.md │ ├── 07_streaming_communication__sse_.md │ ├── 08_multi_agent_orchestration__host_agent_.md │ ├── 09_demo_ui_application___service.md │ └── index.md ├── LangGraph │ ├── 01_graph___stategraph.md │ ├── 02_nodes___pregelnode__.md │ ├── 03_channels.md │ ├── 04_control_flow_primitives___branch____send____interrupt__.md │ ├── 05_pregel_execution_engine.md │ ├── 06_checkpointer___basecheckpointsaver__.md │ └── index.md ├── LevelDB │ ├── 01_table___sstable___tablecache.md │ ├── 02_memtable.md │ ├── 03_write_ahead_log__wal____logwriter_logreader.md │ ├── 04_dbimpl.md │ ├── 05_writebatch.md │ ├── 06_version___versionset.md │ ├── 07_iterator.md │ ├── 08_compaction.md │ ├── 09_internalkey___dbformat.md │ └── index.md ├── MCP Python SDK │ ├── 01_cli___mcp__command_.md │ ├── 02_fastmcp_server___fastmcp__.md │ ├── 03_fastmcp_resources___resource____resourcemanager__.md │ ├── 04_fastmcp_tools___tool____toolmanager__.md │ ├── 05_fastmcp_prompts___prompt____promptmanager__.md │ ├── 06_fastmcp_context___context__.md │ ├── 07_mcp_protocol_types.md │ ├── 08_client_server_sessions___clientsession____serversession__.md │ ├── 09_communication_transports__stdio__sse__websocket__memory_.md │ └── index.md ├── NumPy Core │ ├── 01_ndarray__n_dimensional_array_.md │ ├── 02_dtype__data_type_object_.md │ ├── 03_ufunc__universal_function_.md │ ├── 04_numeric_types___numerictypes__.md │ ├── 05_array_printing___arrayprint__.md │ ├── 06_multiarray_module.md │ ├── 07_umath_module.md │ ├── 08___array_function___protocol___overrides___overrides__.md │ └── index.md ├── OpenManus │ ├── 01_llm.md │ ├── 02_message___memory.md │ ├── 03_baseagent.md │ ├── 04_tool___toolcollection.md │ ├── 05_baseflow.md │ ├── 06_schema.md │ ├── 07_configuration__config_.md │ ├── 08_dockersandbox.md │ ├── 09_mcp__model_context_protocol_.md │ └── index.md ├── PocketFlow │ ├── 01_shared_state___shared__dictionary__.md │ ├── 02_node___basenode____node____asyncnode___.md │ ├── 03_actions___transitions_.md │ ├── 04_flow___flow____asyncflow___.md │ ├── 05_asynchronous_processing___asyncnode____asyncflow___.md │ ├── 06_batch_processing___batchnode____batchflow____asyncparallelbatchnode___.md │ ├── 07_a2a__agent_to_agent__communication_framework_.md │ └── index.md ├── Pydantic Core │ ├── 01_basemodel.md │ ├── 02_fields__fieldinfo___field_function_.md │ ├── 03_configuration__configdict___configwrapper_.md │ ├── 04_custom_logic__decorators___annotated_helpers_.md │ ├── 05_core_schema___validation_serialization.md │ ├── 06_typeadapter.md │ └── index.md ├── Requests │ ├── 01_functional_api.md │ ├── 02_request___response_models.md │ ├── 03_session.md │ ├── 04_cookie_jar.md │ ├── 05_authentication_handlers.md │ ├── 06_exception_hierarchy.md │ ├── 07_transport_adapters.md │ ├── 08_hook_system.md │ └── index.md ├── SmolaAgents │ ├── 01_multistepagent.md │ ├── 02_model_interface.md │ ├── 03_tool.md │ ├── 04_agentmemory.md │ ├── 05_prompttemplates.md │ ├── 06_pythonexecutor.md │ ├── 07_agenttype.md │ ├── 08_agentlogger___monitor.md │ └── index.md ├── _config.yml ├── design.md └── index.md ├── flow.py ├── main.py ├── nodes.py ├── requirements.txt └── utils ├── __init__.py ├── call_llm.py ├── crawl_github_files.py └── crawl_local_files.py /.dockerignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / cache files 2 | __pycache__/ 3 | *.py[cod] 4 | *.pyo 5 | *.pyd 6 | 7 | # Virtual environments 8 | venv/ 9 | env/ 10 | .venv/ 11 | .env/ 12 | 13 | # Distribution / packaging 14 | *.egg-info/ 15 | build/ 16 | dist/ 17 | 18 | # Git and other VCS 19 | .git/ 20 | .gitignore 21 | 22 | # Editor files 23 | *.swp 24 | *.swo 25 | *.bak 26 | *.tmp 27 | .DS_Store 28 | .idea/ 29 | .vscode/ 30 | 31 | # Secrets (if you’re using .env for API keys etc.) 32 | .env 33 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | GEMINI_PROJECT_ID= 2 | GEMINI_API_KEY= 3 | GITHUB_TOKEN= 4 | OPENROUTER_API_KEY = 5 | OPENROUTER_MODEL = -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | node_modules/ 3 | vendor/ 4 | .pnp/ 5 | .pnp.js 6 | 7 | # Build outputs 8 | dist/ 9 | build/ 10 | out/ 11 | *.pyc 12 | __pycache__/ 13 | 14 | # Environment files 15 | .env 16 | .env.local 17 | .env.*.local 18 | .env.development 19 | .env.test 20 | .env.production 21 | 22 | # Python virtual environments 23 | .venv/ 24 | venv/ 25 | 26 | # IDE - VSCode 27 | .vscode/* 28 | !.vscode/settings.json 29 | !.vscode/tasks.json 30 | !.vscode/launch.json 31 | !.vscode/extensions.json 32 | 33 | # IDE - JetBrains 34 | .idea/ 35 | *.iml 36 | *.iws 37 | *.ipr 38 | 39 | # IDE - Eclipse 40 | .project 41 | .classpath 42 | .settings/ 43 | 44 | # Logs 45 | logs/ 46 | *.log 47 | npm-debug.log* 48 | yarn-debug.log* 49 | yarn-error.log* 50 | 51 | # Operating System 52 | .DS_Store 53 | Thumbs.db 54 | *.swp 55 | *.swo 56 | 57 | # Testing 58 | coverage/ 59 | .nyc_output/ 60 | 61 | # Temporary files 62 | *.tmp 63 | *.temp 64 | .cache/ 65 | 66 | # Compiled files 67 | *.com 68 | *.class 69 | *.dll 70 | *.exe 71 | *.o 72 | *.so 73 | 74 | # Package files 75 | *.7z 76 | *.dmg 77 | *.gz 78 | *.iso 79 | *.jar 80 | *.rar 81 | *.tar 82 | *.zip 83 | 84 | # Database 85 | *.sqlite 86 | *.sqlite3 87 | *.db 88 | 89 | # Optional npm cache directory 90 | .npm 91 | 92 | # Optional eslint cache 93 | .eslintcache 94 | 95 | # Optional REPL history 96 | .node_repl_history 97 | 98 | # LLM cache 99 | llm_cache.json 100 | 101 | # Output files 102 | output/ 103 | 104 | # uv manage 105 | pyproject.toml 106 | uv.lock 107 | 108 | docs/*.pdf 109 | docs/design-cn.md 110 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10-slim 2 | 3 | # update packages, install git and remove cache 4 | RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* 5 | 6 | WORKDIR /app 7 | 8 | COPY requirements.txt . 9 | RUN pip install --no-cache-dir -r requirements.txt 10 | 11 | COPY . . 12 | 13 | ENTRYPOINT ["python", "main.py"] 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Zachary Huang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

Turns Codebase into Easy Tutorial with AI

2 | 3 | ![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) 4 | 5 | 6 | 7 | > *Ever stared at a new codebase written by others feeling completely lost? This tutorial shows you how to build an AI agent that analyzes GitHub repositories and creates beginner-friendly tutorials explaining exactly how the code works.* 8 | 9 |

10 | 13 |

14 | 15 | This is a tutorial project of [Pocket Flow](https://github.com/The-Pocket/PocketFlow), a 100-line LLM framework. It crawls GitHub repositories and builds a knowledge base from the code. It analyzes entire codebases to identify core abstractions and how they interact, and transforms complex code into beginner-friendly tutorials with clear visualizations. 16 | 17 | - Check out the [YouTube Development Tutorial](https://youtu.be/AFY67zOpbSo) for more! 18 | 19 | - Check out the [Substack Post Tutorial](https://zacharyhuang.substack.com/p/ai-codebase-knowledge-builder-full) for more! 20 | 21 |   **🔸 🎉 Reached Hacker News Front Page** (April 2025) with >900 up‑votes: [Discussion »](https://news.ycombinator.com/item?id=43739456) 22 | 23 |   **🔸 🎊 Online Service Now Live!** (May 2025) Try our new online version at [https://code2tutorial.com/](https://code2tutorial.com/) – just paste a GitHub link, no installation needed! 24 | 25 | ## ⭐ Example Results for Popular GitHub Repositories! 26 | 27 |

28 | 31 |

32 | 33 | 🤯 All these tutorials are generated **entirely by AI** by crawling the GitHub repo! 34 | 35 | - [AutoGen Core](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/AutoGen%20Core) - Build AI teams that talk, think, and solve problems together like coworkers! 36 | 37 | - [Browser Use](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/Browser%20Use) - Let AI surf the web for you, clicking buttons and filling forms like a digital assistant! 38 | 39 | - [Celery](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/Celery) - Supercharge your app with background tasks that run while you sleep! 40 | 41 | - [Click](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/Click) - Turn Python functions into slick command-line tools with just a decorator! 42 | 43 | - [Codex](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/Codex) - Turn plain English into working code with this AI terminal wizard! 44 | 45 | - [Crawl4AI](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/Crawl4AI) - Train your AI to extract exactly what matters from any website! 46 | 47 | - [CrewAI](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/CrewAI) - Assemble a dream team of AI specialists to tackle impossible problems! 48 | 49 | - [DSPy](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/DSPy) - Build LLM apps like Lego blocks that optimize themselves! 50 | 51 | - [FastAPI](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/FastAPI) - Create APIs at lightning speed with automatic docs that clients will love! 52 | 53 | - [Flask](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/Flask) - Craft web apps with minimal code that scales from prototype to production! 54 | 55 | - [Google A2A](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/Google%20A2A) - The universal language that lets AI agents collaborate across borders! 56 | 57 | - [LangGraph](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/LangGraph) - Design AI agents as flowcharts where each step remembers what happened before! 58 | 59 | - [LevelDB](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/LevelDB) - Store data at warp speed with Google's engine that powers blockchains! 60 | 61 | - [MCP Python SDK](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/MCP%20Python%20SDK) - Build powerful apps that communicate through an elegant protocol without sweating the details! 62 | 63 | - [NumPy Core](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/NumPy%20Core) - Master the engine behind data science that makes Python as fast as C! 64 | 65 | - [OpenManus](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/OpenManus) - Build AI agents with digital brains that think, learn, and use tools just like humans do! 66 | 67 | - [PocketFlow](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/PocketFlow) - 100-line LLM framework. Let Agents build Agents! 68 | 69 | - [Pydantic Core](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/Pydantic%20Core) - Validate data at rocket speed with just Python type hints! 70 | 71 | - [Requests](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/Requests) - Talk to the internet in Python with code so simple it feels like cheating! 72 | 73 | - [SmolaAgents](https://the-pocket.github.io/PocketFlow-Tutorial-Codebase-Knowledge/SmolaAgents) - Build tiny AI agents that punch way above their weight class! 74 | 75 | - Showcase Your AI-Generated Tutorials in [Discussions](https://github.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge/discussions)! 76 | 77 | ## 🚀 Getting Started 78 | 79 | 1. Clone this repository 80 | ```bash 81 | git clone https://github.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge 82 | ``` 83 | 84 | 3. Install dependencies: 85 | ```bash 86 | pip install -r requirements.txt 87 | ``` 88 | 89 | 4. Set up LLM in [`utils/call_llm.py`](./utils/call_llm.py) by providing credentials. By default, you can use the [AI Studio key](https://aistudio.google.com/app/apikey) with this client for Gemini Pro 2.5: 90 | 91 | ```python 92 | client = genai.Client( 93 | api_key=os.getenv("GEMINI_API_KEY", "your-api_key"), 94 | ) 95 | ``` 96 | 97 | You can use your own models. We highly recommend the latest models with thinking capabilities (Claude 3.7 with thinking, O1). You can verify that it is correctly set up by running: 98 | ```bash 99 | python utils/call_llm.py 100 | ``` 101 | 102 | 5. Generate a complete codebase tutorial by running the main script: 103 | ```bash 104 | # Analyze a GitHub repository 105 | python main.py --repo https://github.com/username/repo --include "*.py" "*.js" --exclude "tests/*" --max-size 50000 106 | 107 | # Or, analyze a local directory 108 | python main.py --dir /path/to/your/codebase --include "*.py" --exclude "*test*" 109 | 110 | # Or, generate a tutorial in Chinese 111 | python main.py --repo https://github.com/username/repo --language "Chinese" 112 | ``` 113 | 114 | - `--repo` or `--dir` - Specify either a GitHub repo URL or a local directory path (required, mutually exclusive) 115 | - `-n, --name` - Project name (optional, derived from URL/directory if omitted) 116 | - `-t, --token` - GitHub token (or set GITHUB_TOKEN environment variable) 117 | - `-o, --output` - Output directory (default: ./output) 118 | - `-i, --include` - Files to include (e.g., "`*.py`" "`*.js`") 119 | - `-e, --exclude` - Files to exclude (e.g., "`tests/*`" "`docs/*`") 120 | - `-s, --max-size` - Maximum file size in bytes (default: 100KB) 121 | - `--language` - Language for the generated tutorial (default: "english") 122 | - `--max-abstractions` - Maximum number of abstractions to identify (default: 10) 123 | - `--no-cache` - Disable LLM response caching (default: caching enabled) 124 | 125 | The application will crawl the repository, analyze the codebase structure, generate tutorial content in the specified language, and save the output in the specified directory (default: ./output). 126 | 127 | 128 |
129 | 130 | 🐳 Running with Docker 131 | 132 | To run this project in a Docker container, you'll need to pass your API keys as environment variables. 133 | 134 | 1. Build the Docker image 135 | ```bash 136 | docker build -t pocketflow-app . 137 | ``` 138 | 139 | 2. Run the container 140 | 141 | You'll need to provide your `GEMINI_API_KEY` for the LLM to function. If you're analyzing private GitHub repositories or want to avoid rate limits, also provide your `GITHUB_TOKEN`. 142 | 143 | Mount a local directory to `/app/output` inside the container to access the generated tutorials on your host machine. 144 | 145 | **Example for analyzing a public GitHub repository:** 146 | 147 | ```bash 148 | docker run -it --rm \ 149 | -e GEMINI_API_KEY="YOUR_GEMINI_API_KEY_HERE" \ 150 | -v "$(pwd)/output_tutorials":/app/output \ 151 | pocketflow-app --repo https://github.com/username/repo 152 | ``` 153 | 154 | **Example for analyzing a local directory:** 155 | 156 | ```bash 157 | docker run -it --rm \ 158 | -e GEMINI_API_KEY="YOUR_GEMINI_API_KEY_HERE" \ 159 | -v "/path/to/your/local_codebase":/app/code_to_analyze \ 160 | -v "$(pwd)/output_tutorials":/app/output \ 161 | pocketflow-app --dir /app/code_to_analyze 162 | ``` 163 |
164 | 165 | ## 💡 Development Tutorial 166 | 167 | - I built using [**Agentic Coding**](https://zacharyhuang.substack.com/p/agentic-coding-the-most-fun-way-to), the fastest development paradigm, where humans simply [design](docs/design.md) and agents [code](flow.py). 168 | 169 | - The secret weapon is [Pocket Flow](https://github.com/The-Pocket/PocketFlow), a 100-line LLM framework that lets Agents (e.g., Cursor AI) build for you 170 | 171 | - Check out the Step-by-step YouTube development tutorial: 172 | 173 |
174 |
175 | 176 | Pocket Flow Codebase Tutorial 177 | 178 |
179 |
180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /assets/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge/755e76733dfd257997bc039727697c8000b38aa6/assets/banner.png -------------------------------------------------------------------------------- /assets/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge/755e76733dfd257997bc039727697c8000b38aa6/assets/example.png -------------------------------------------------------------------------------- /assets/youtube_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge/755e76733dfd257997bc039727697c8000b38aa6/assets/youtube_thumbnail.png -------------------------------------------------------------------------------- /docs/AutoGen Core/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "AutoGen Core" 4 | nav_order: 3 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: AutoGen Core 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | AutoGen Core[View Repo](https://github.com/microsoft/autogen/tree/e45a15766746d95f8cfaaa705b0371267bec812e/python/packages/autogen-core/src/autogen_core) helps you build applications with multiple **_Agents_** that can work together. 13 | Think of it like creating a team of specialized workers (*Agents*) who can communicate and use tools to solve problems. 14 | The **_AgentRuntime_** acts as the manager, handling messages and agent lifecycles. 15 | Agents communicate using a **_Messaging System_** (Topics and Subscriptions), can use **_Tools_** for specific tasks, interact with language models via a **_ChatCompletionClient_** while managing conversation history with **_ChatCompletionContext_**, and remember information using **_Memory_**. 16 | **_Components_** provide a standard way to define and configure these building blocks. 17 | 18 | 19 | ```mermaid 20 | flowchart TD 21 | A0["0: Agent"] 22 | A1["1: AgentRuntime"] 23 | A2["2: Messaging System (Topic & Subscription)"] 24 | A3["3: Component"] 25 | A4["4: Tool"] 26 | A5["5: ChatCompletionClient"] 27 | A6["6: ChatCompletionContext"] 28 | A7["7: Memory"] 29 | A1 -- "Manages lifecycle" --> A0 30 | A1 -- "Uses for message routing" --> A2 31 | A0 -- "Uses LLM client" --> A5 32 | A0 -- "Executes tools" --> A4 33 | A0 -- "Accesses memory" --> A7 34 | A5 -- "Gets history from" --> A6 35 | A5 -- "Uses tool schema" --> A4 36 | A7 -- "Updates LLM context" --> A6 37 | A4 -- "Implemented as" --> A3 38 | ``` 39 | -------------------------------------------------------------------------------- /docs/Browser Use/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Browser Use" 4 | nav_order: 4 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: Browser Use 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | **Browser Use**[View Repo](https://github.com/browser-use/browser-use/tree/3076ba0e83f30b45971af58fe2aeff64472da812/browser_use) is a project that allows an *AI agent* to control a web browser and perform tasks automatically. 13 | Think of it like an AI assistant that can browse websites, fill forms, click buttons, and extract information based on your instructions. It uses a Large Language Model (LLM) as its "brain" to decide what actions to take on a webpage to complete a given *task*. The project manages the browser session, understands the page structure (DOM), and communicates back and forth with the LLM. 14 | 15 | ```mermaid 16 | flowchart TD 17 | A0["Agent"] 18 | A1["BrowserContext"] 19 | A2["Action Controller & Registry"] 20 | A3["DOM Representation"] 21 | A4["Message Manager"] 22 | A5["System Prompt"] 23 | A6["Data Structures (Views)"] 24 | A7["Telemetry Service"] 25 | A0 -- "Gets state from" --> A1 26 | A0 -- "Uses to execute actions" --> A2 27 | A0 -- "Uses for LLM communication" --> A4 28 | A0 -- "Gets instructions from" --> A5 29 | A0 -- "Uses/Produces data formats" --> A6 30 | A0 -- "Logs events to" --> A7 31 | A1 -- "Gets DOM structure via" --> A3 32 | A1 -- "Provides BrowserState" --> A6 33 | A2 -- "Executes actions on" --> A1 34 | A2 -- "Defines/Uses ActionModel/Ac..." --> A6 35 | A2 -- "Logs registered functions to" --> A7 36 | A3 -- "Provides structure to" --> A1 37 | A3 -- "Uses DOM structures" --> A6 38 | A4 -- "Provides messages to" --> A0 39 | A4 -- "Initializes with" --> A5 40 | A4 -- "Formats data using" --> A6 41 | A5 -- "Defines structure for Agent..." --> A6 42 | A7 -- "Receives events from" --> A0 43 | ``` 44 | -------------------------------------------------------------------------------- /docs/Celery/04_broker_connection__amqp_.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Broker Connection (AMQP)" 4 | parent: "Celery" 5 | nav_order: 4 6 | --- 7 | 8 | # Chapter 4: Broker Connection (AMQP) - Celery's Postal Service 9 | 10 | In [Chapter 3: Task](03_task.md), we learned how to define "job descriptions" (Tasks) like `add(x, y)` and how to request them using `.delay()`. But when you call `add.delay(2, 2)`, how does that request actually *get* to a worker process that can perform the addition? It doesn't just magically appear! 11 | 12 | This is where the **Broker Connection** comes in. Think of it as Celery's built-in postal service. 13 | 14 | ## What Problem Does the Broker Connection Solve? 15 | 16 | Imagine you want to send a letter (a task request) to a friend (a worker) who lives in another city. You can't just shout the message out your window and hope they hear it. You need: 17 | 18 | 1. A **Post Office** (the Message Broker, like RabbitMQ or Redis) that handles mail. 19 | 2. A way to **talk to the Post Office** (the Broker Connection) to drop off your letter or pick up mail addressed to you. 20 | 21 | The Broker Connection is that crucial link between your application (where you call `.delay()`) or your Celery worker and the message broker system. It manages sending messages *to* the broker and receiving messages *from* the broker reliably. 22 | 23 | Without this connection, your task requests would never leave your application, and your workers would never know there's work waiting for them. 24 | 25 | ## Key Concepts: Post Office & Rules 26 | 27 | Let's break down the pieces: 28 | 29 | 1. **The Message Broker (The Post Office):** This is a separate piece of software that acts as a central hub for messages. Common choices are RabbitMQ and Redis. You tell Celery its address using the `broker_url` setting in your [Configuration](02_configuration.md). 30 | ```python 31 | # From Chapter 2 - celeryconfig.py 32 | broker_url = 'amqp://guest:guest@localhost:5672//' # Example for RabbitMQ 33 | # Or maybe: broker_url = 'redis://localhost:6379/0' # Example for Redis 34 | ``` 35 | 36 | 2. **The Connection (Talking to the Staff):** This is the active communication channel established between your Python code (either your main app or a worker) and the broker. It's like having an open phone line to the post office. Celery, using a library called `kombu`, handles creating and managing these connections based on the `broker_url`. 37 | 38 | 3. **AMQP (The Postal Rules):** AMQP stands for **Advanced Message Queuing Protocol**. Think of it as a specific set of rules and procedures for how post offices should operate – how letters should be addressed, sorted, delivered, and confirmed. 39 | * RabbitMQ is a broker that speaks AMQP natively. 40 | * Other brokers, like Redis, use different protocols (their own set of rules). 41 | * **Why mention AMQP?** It's a very common and powerful protocol for message queuing, and the principles behind it (exchanges, queues, routing) are fundamental to how Celery routes tasks, even when using other brokers. Celery's internal component for handling this communication is often referred to as `app.amqp` (found in `app/amqp.py`), even though the underlying library (`kombu`) supports multiple protocols. So, we focus on the *concept* of managing the broker connection, often using AMQP terminology as a reference point. 42 | 43 | 4. **Producer (Sending Mail):** When your application calls `add.delay(2, 2)`, it acts as a *producer*. It uses its broker connection to send a message ("Please run 'add' with arguments (2, 2)") to the broker. 44 | 45 | 5. **Consumer (Receiving Mail):** A Celery [Worker](05_worker.md) acts as a *consumer*. It uses its *own* broker connection to constantly check a specific mailbox (queue) at the broker for new messages. When it finds one, it takes it, performs the task, and tells the broker it's done. 46 | 47 | ## How Sending a Task Uses the Connection 48 | 49 | Let's revisit sending a task from [Chapter 3: Task](03_task.md): 50 | 51 | ```python 52 | # run_tasks.py (simplified) 53 | from tasks import add 54 | from celery_app import app # Assume app is configured with a broker_url 55 | 56 | # 1. You call .delay() 57 | print("Sending task...") 58 | result_promise = add.delay(2, 2) 59 | # Behind the scenes: 60 | # a. Celery looks at the 'add' task, finds its associated 'app'. 61 | # b. It asks 'app' for the broker_url from its configuration. 62 | # c. It uses the app.amqp component (powered by Kombu) to get a connection 63 | # to the broker specified by the URL (e.g., 'amqp://localhost...'). 64 | # d. It packages the task name 'tasks.add' and args (2, 2) into a message. 65 | # e. It uses the connection to 'publish' (send) the message to the broker. 66 | 67 | print(f"Task sent! ID: {result_promise.id}") 68 | ``` 69 | 70 | The `add.delay(2, 2)` call triggers this whole process. It needs the configured `broker_url` to know *which* post office to connect to, and the broker connection handles the actual sending of the "letter" (task message). 71 | 72 | Similarly, a running Celery [Worker](05_worker.md) establishes its own connection to the *same* broker. It uses this connection to *listen* for incoming messages on the queues it's assigned to. 73 | 74 | ## How It Works Internally (Simplified) 75 | 76 | Celery uses a powerful library called **Kombu** to handle the low-level details of connecting and talking to different types of brokers (RabbitMQ, Redis, etc.). The `app.amqp` object in Celery acts as a high-level interface to Kombu's features. 77 | 78 | 1. **Configuration:** The `broker_url` tells Kombu where and how to connect. 79 | 2. **Connection Pool:** To be efficient, Celery (via Kombu) often maintains a *pool* of connections. When you send a task, it might grab an existing, idle connection from the pool instead of creating a new one every time. This is faster. You can see this managed by `app.producer_pool` in `app/base.py`. 80 | 3. **Producer:** When `task.delay()` is called, it ultimately uses a `kombu.Producer` object. This object represents the ability to *send* messages. It's tied to a specific connection and channel. 81 | 4. **Publishing:** The producer's `publish()` method is called. This takes the task message (already serialized into a format like JSON), specifies the destination (exchange and routing key - think of these like the address and sorting code on an envelope), and sends it over the connection to the broker. 82 | 5. **Consumer:** A Worker uses a `kombu.Consumer` object. This object is set up to listen on specific queues via its connection. When a message arrives in one of those queues, the broker pushes it to the consumer over the connection, and the consumer triggers the appropriate Celery task execution logic. 83 | 84 | ```mermaid 85 | sequenceDiagram 86 | participant Client as Your App Code 87 | participant Task as add.delay() 88 | participant App as Celery App 89 | participant AppAMQP as app.amqp (Kombu Interface) 90 | participant Broker as RabbitMQ / Redis 91 | 92 | Client->>Task: Call add.delay(2, 2) 93 | Task->>App: Get broker config (broker_url) 94 | App-->>Task: broker_url 95 | Task->>App: Ask to send task 'tasks.add' 96 | App->>AppAMQP: Send task message('tasks.add', (2, 2), ...) 97 | Note over AppAMQP: Gets connection/producer (maybe from pool) 98 | AppAMQP->>Broker: publish(message, routing_info) via Connection 99 | Broker-->>AppAMQP: Acknowledge message received 100 | AppAMQP-->>App: Message sent successfully 101 | App-->>Task: Return AsyncResult 102 | Task-->>Client: Return AsyncResult 103 | ``` 104 | 105 | This shows the flow: your code calls `.delay()`, Celery uses its configured connection details (`app.amqp` layer) to get a connection and producer, and then publishes the message to the broker. 106 | 107 | ## Code Dive: Sending a Message 108 | 109 | Let's peek inside `app/amqp.py` where the `AMQP` class orchestrates sending. The `send_task_message` method (simplified below) is key. 110 | 111 | ```python 112 | # Simplified from app/amqp.py within the AMQP class 113 | 114 | # This function is configured internally and gets called by app.send_task 115 | def _create_task_sender(self): 116 | # ... (lots of setup: getting defaults from config, signals) ... 117 | default_serializer = self.app.conf.task_serializer 118 | default_compressor = self.app.conf.task_compression 119 | 120 | def send_task_message(producer, name, message, 121 | exchange=None, routing_key=None, queue=None, 122 | serializer=None, compression=None, declare=None, 123 | retry=None, retry_policy=None, 124 | **properties): 125 | # ... (Determine exchange, routing_key, queue based on config/options) ... 126 | # ... (Prepare headers, properties, handle retries) ... 127 | 128 | headers, properties, body, sent_event = message # Unpack the prepared message tuple 129 | 130 | # The core action: Use the producer to publish the message! 131 | ret = producer.publish( 132 | body, # The actual task payload (args, kwargs, etc.) 133 | exchange=exchange, 134 | routing_key=routing_key, 135 | serializer=serializer or default_serializer, # e.g., 'json' 136 | compression=compression or default_compressor, 137 | retry=retry, 138 | retry_policy=retry_policy, 139 | declare=declare, # Maybe declare queues/exchanges if needed 140 | headers=headers, 141 | **properties # Other message properties (correlation_id, etc.) 142 | ) 143 | 144 | # ... (Send signals like task_sent, publish events if configured) ... 145 | return ret 146 | return send_task_message 147 | ``` 148 | 149 | **Explanation:** 150 | 151 | * This function takes a `producer` object (which is linked to a broker connection via Kombu). 152 | * It figures out the final destination details (exchange, routing key). 153 | * It calls `producer.publish()`, passing the task body and all the necessary options (like serializer). This is the function that actually sends the data over the network connection to the broker. 154 | 155 | The `Connection` objects themselves are managed by Kombu (see `kombu/connection.py`). Celery uses these objects via its `app.connection_for_write()` or `app.connection_for_read()` methods, which often pull from the connection pool (`kombu.pools`). 156 | 157 | ## Conclusion 158 | 159 | The Broker Connection is Celery's vital communication link, its "postal service." 160 | 161 | * It connects your application and workers to the **Message Broker** (like RabbitMQ or Redis). 162 | * It uses the `broker_url` from your [Configuration](02_configuration.md) to know where to connect. 163 | * Protocols like **AMQP** define the "rules" for communication, although Celery's underlying library (Kombu) handles various protocols. 164 | * Your app **produces** task messages and sends them over the connection. 165 | * Workers **consume** task messages received over their connection. 166 | * Celery manages connections efficiently, often using **pools**. 167 | 168 | Understanding the broker connection helps clarify how tasks move from where they're requested to where they run. Now that we know how tasks are defined and sent across the wire, let's look at the entity that actually picks them up and does the work. 169 | 170 | **Next:** [Chapter 5: Worker](05_worker.md) 171 | 172 | --- 173 | 174 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/Celery/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Celery" 4 | nav_order: 5 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: Celery 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | Celery[View Repo](https://github.com/celery/celery/tree/d1c35bbdf014f13f4ab698d75e3ea381a017b090/celery) is a system for running **distributed tasks** *asynchronously*. You define *units of work* (Tasks) in your Python code. When you want a task to run, you send a message using a **message broker** (like RabbitMQ or Redis). One or more **Worker** processes are running in the background, listening for these messages. When a worker receives a message, it executes the corresponding task. Optionally, the task's result (or any error) can be stored in a **Result Backend** (like Redis or a database) so you can check its status or retrieve the output later. Celery helps manage this whole process, making it easier to handle background jobs, scheduled tasks, and complex workflows. 13 | 14 | ```mermaid 15 | flowchart TD 16 | A0["Celery App"] 17 | A1["Task"] 18 | A2["Worker"] 19 | A3["Broker Connection (AMQP)"] 20 | A4["Result Backend"] 21 | A5["Canvas (Signatures & Primitives)"] 22 | A6["Beat (Scheduler)"] 23 | A7["Configuration"] 24 | A8["Events"] 25 | A9["Bootsteps"] 26 | A0 -- "Defines and sends" --> A1 27 | A0 -- "Uses for messaging" --> A3 28 | A0 -- "Uses for results" --> A4 29 | A0 -- "Loads and uses" --> A7 30 | A1 -- "Updates state in" --> A4 31 | A2 -- "Executes" --> A1 32 | A2 -- "Fetches tasks from" --> A3 33 | A2 -- "Uses for lifecycle" --> A9 34 | A5 -- "Represents task invocation" --> A1 35 | A6 -- "Sends scheduled tasks via" --> A3 36 | A8 -- "Sends events via" --> A3 37 | A9 -- "Manages connection via" --> A3 38 | ``` 39 | 40 | -------------------------------------------------------------------------------- /docs/Click/01_command___group.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Command & Group" 4 | parent: "Click" 5 | nav_order: 1 6 | --- 7 | 8 | # Chapter 1: Commands and Groups: The Building Blocks 9 | 10 | Welcome to your first step in learning Click! Imagine you want to create your own command-line tool, maybe something like `git` or `docker`. How do you tell your program what to do when someone types `git commit` or `docker build`? That's where **Commands** and **Groups** come in. They are the fundamental building blocks for any Click application. 11 | 12 | Think about a simple tool. Maybe you want a program that can greet someone. You'd type `greet Alice` in your terminal, and it would print "Hello Alice!". In Click, this single action, "greet", would be represented by a `Command`. 13 | 14 | Now, what if your tool needed to do *more* than one thing? Maybe besides greeting, it could also say goodbye. You might want to type `mytool greet Alice` or `mytool goodbye Bob`. The main `mytool` part acts like a container or a menu, holding the different actions (`greet`, `goodbye`). This container is what Click calls a `Group`. 15 | 16 | So: 17 | 18 | * `Command`: Represents a single action your tool can perform. 19 | * `Group`: Represents a collection of related actions (Commands or other Groups). 20 | 21 | Let's dive in and see how to create them! 22 | 23 | ## Your First Command 24 | 25 | Creating a command in Click is surprisingly simple. You basically write a normal Python function and then "decorate" it to tell Click it's a command-line command. 26 | 27 | Let's make a command that just prints "Hello World!". 28 | 29 | ```python 30 | # hello_app.py 31 | import click 32 | 33 | @click.command() 34 | def hello(): 35 | """A simple command that says Hello World""" 36 | print("Hello World!") 37 | 38 | if __name__ == '__main__': 39 | hello() 40 | ``` 41 | 42 | Let's break this down: 43 | 44 | 1. `import click`: We need to import the Click library first. 45 | 2. `@click.command()`: This is the magic part! It's called a decorator. It transforms the Python function `hello()` right below it into a Click `Command` object. We'll learn more about [Decorators](02_decorators.md) in the next chapter, but for now, just know this line turns `hello` into something Click understands as a command. 46 | 3. `def hello(): ...`: This is a standard Python function. The code inside this function is what will run when you execute the command from your terminal. 47 | 4. `"""A simple command that says Hello World"""`: This is a docstring. Click cleverly uses the function's docstring as the help text for the command! 48 | 5. `if __name__ == '__main__': hello()`: This standard Python construct checks if the script is being run directly. If it is, it calls our `hello` command function (which is now actually a Click `Command` object). 49 | 50 | **Try running it!** Save the code above as `hello_app.py`. Open your terminal in the same directory and run: 51 | 52 | ```bash 53 | $ python hello_app.py 54 | Hello World! 55 | ``` 56 | 57 | It works! You just created your first command-line command with Click. 58 | 59 | **Bonus: Automatic Help!** 60 | 61 | Click automatically generates help screens for you. Try running your command with `--help`: 62 | 63 | ```bash 64 | $ python hello_app.py --help 65 | Usage: hello_app.py [OPTIONS] 66 | 67 | A simple command that says Hello World 68 | 69 | Options: 70 | --help Show this message and exit. 71 | ``` 72 | 73 | See? Click used the docstring we wrote (`A simple command that says Hello World`) and added a standard `--help` option for free! 74 | 75 | ## Grouping Commands 76 | 77 | Okay, one command is nice, but real tools often have multiple commands. Like `git` has `commit`, `pull`, `push`, etc. Let's say we want our tool to have two commands: `hello` and `goodbye`. 78 | 79 | We need a way to group these commands together. That's what `click.group()` is for. A `Group` acts as the main entry point and can have other commands attached to it. 80 | 81 | ```python 82 | # multi_app.py 83 | import click 84 | 85 | # 1. Create the main group 86 | @click.group() 87 | def cli(): 88 | """A simple tool with multiple commands.""" 89 | pass # The group function itself doesn't need to do anything 90 | 91 | # 2. Define the 'hello' command 92 | @click.command() 93 | def hello(): 94 | """Says Hello World""" 95 | print("Hello World!") 96 | 97 | # 3. Define the 'goodbye' command 98 | @click.command() 99 | def goodbye(): 100 | """Says Goodbye World""" 101 | print("Goodbye World!") 102 | 103 | # 4. Attach the commands to the group 104 | cli.add_command(hello) 105 | cli.add_command(goodbye) 106 | 107 | if __name__ == '__main__': 108 | cli() # Run the main group 109 | ``` 110 | 111 | What's changed? 112 | 113 | 1. We created a function `cli` and decorated it with `@click.group()`. This makes `cli` our main entry point, a container for other commands. Notice the function body is just `pass` – often, the group function itself doesn't need logic; its job is to hold other commands. 114 | 2. We defined `hello` and `goodbye` just like before, using `@click.command()`. 115 | 3. Crucially, we *attached* our commands to the group: `cli.add_command(hello)` and `cli.add_command(goodbye)`. This tells Click that `hello` and `goodbye` are subcommands of `cli`. 116 | 4. Finally, in the `if __name__ == '__main__':` block, we run `cli()`, our main group. 117 | 118 | **Let's run this!** Save it as `multi_app.py`. 119 | 120 | First, check the main help screen: 121 | 122 | ```bash 123 | $ python multi_app.py --help 124 | Usage: multi_app.py [OPTIONS] COMMAND [ARGS]... 125 | 126 | A simple tool with multiple commands. 127 | 128 | Options: 129 | --help Show this message and exit. 130 | 131 | Commands: 132 | goodbye Says Goodbye World 133 | hello Says Hello World 134 | ``` 135 | 136 | Look! Click now lists `goodbye` and `hello` under "Commands". It automatically figured out their names from the function names (`goodbye`, `hello`) and their help text from their docstrings. 137 | 138 | Now, run the specific commands: 139 | 140 | ```bash 141 | $ python multi_app.py hello 142 | Hello World! 143 | 144 | $ python multi_app.py goodbye 145 | Goodbye World! 146 | ``` 147 | 148 | You've successfully created a multi-command CLI tool! 149 | 150 | *(Self-promotion: There's an even shorter way to attach commands using decorators directly on the group, which we'll see in [Decorators](02_decorators.md)!)* 151 | 152 | ## How It Works Under the Hood 153 | 154 | What's really happening when you use `@click.command()` or `@click.group()`? 155 | 156 | 1. **Decoration:** The decorator (`@click.command` or `@click.group`) takes your Python function (`hello`, `goodbye`, `cli`). It wraps this function inside a Click object – either a `Command` instance or a `Group` instance (which is actually a special type of `Command`). These objects store your original function as the `callback` to be executed later. They also store metadata like the command name (derived from the function name) and the help text (from the docstring). You can find the code for these decorators in `decorators.py` and the `Command`/`Group` classes in `core.py`. 157 | 158 | 2. **Execution:** When you run `python multi_app.py hello`, Python executes the `cli()` call at the bottom. Since `cli` is a `Group` object created by Click, it knows how to parse the command-line arguments (`hello` in this case). 159 | 160 | 3. **Parsing & Dispatch:** The `cli` group looks at the first argument (`hello`). It checks its list of registered subcommands (which we added using `cli.add_command`). It finds a match with the `hello` command object. 161 | 162 | 4. **Callback:** The `cli` group then invokes the `hello` command object. The `hello` command object, in turn, calls the original Python function (`hello()`) that it stored earlier as its `callback`. 163 | 164 | Here's a simplified view of what happens when you run `python multi_app.py hello`: 165 | 166 | ```mermaid 167 | sequenceDiagram 168 | participant User 169 | participant Terminal 170 | participant PythonScript (multi_app.py) 171 | participant ClickRuntime 172 | participant cli_Group as cli (Group Object) 173 | participant hello_Command as hello (Command Object) 174 | 175 | User->>Terminal: python multi_app.py hello 176 | Terminal->>PythonScript: Executes script with args ["hello"] 177 | PythonScript->>ClickRuntime: Calls cli() entry point 178 | ClickRuntime->>cli_Group: Asks to handle args ["hello"] 179 | cli_Group->>cli_Group: Parses args, identifies "hello" as subcommand 180 | cli_Group->>hello_Command: Invokes the 'hello' command 181 | hello_Command->>hello_Command: Executes its callback (the original hello() function) 182 | hello_Command-->>PythonScript: Prints "Hello World!" 183 | PythonScript-->>Terminal: Shows output 184 | Terminal-->>User: Displays "Hello World!" 185 | ``` 186 | 187 | This process of parsing arguments and calling the right function based on the command structure is the core job of Click, making it easy for *you* to just focus on writing the functions for each command. 188 | 189 | ## Conclusion 190 | 191 | You've learned about the two most fundamental concepts in Click: 192 | 193 | * `Command`: Represents a single action, created by decorating a function with `@click.command()`. 194 | * `Group`: Acts as a container for multiple commands (or other groups), created with `@click.group()`. Groups allow you to structure your CLI application logically. 195 | 196 | We saw how Click uses decorators to transform simple Python functions into powerful command-line interface components, automatically handling things like help text generation and command dispatching. 197 | 198 | Commands and Groups form the basic structure, but how do we pass information *into* our commands (like `git commit -m "My message"`)? And what other cool things can decorators do? We'll explore that starting with a deeper look at decorators in the next chapter! 199 | 200 | Next up: [Chapter 2: Decorators](02_decorators.md) 201 | 202 | --- 203 | 204 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/Click/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Click" 4 | nav_order: 6 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: Click 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | Click[View Repo](https://github.com/pallets/click/tree/main/src/click) is a Python library that makes creating **command-line interfaces (CLIs)** *easy and fun*. 13 | It uses simple Python **decorators** (`@click.command`, `@click.option`, etc.) to turn your functions into CLI commands with options and arguments. 14 | Click handles parsing user input, generating help messages, validating data types, and managing the flow between commands, letting you focus on your application's logic. 15 | It also provides tools for *terminal interactions* like prompting users and showing progress bars. 16 | 17 | 18 | ```mermaid 19 | flowchart TD 20 | A0["Context"] 21 | A1["Command / Group"] 22 | A2["Parameter (Option / Argument)"] 23 | A3["ParamType"] 24 | A4["Decorators"] 25 | A5["Term UI (Terminal User Interface)"] 26 | A6["Click Exceptions"] 27 | A4 -- "Creates/Configures" --> A1 28 | A4 -- "Creates/Configures" --> A2 29 | A0 -- "Manages execution of" --> A1 30 | A0 -- "Holds parsed values for" --> A2 31 | A2 -- "Uses for validation/conversion" --> A3 32 | A3 -- "Raises on conversion error" --> A6 33 | A1 -- "Uses for user interaction" --> A5 34 | A0 -- "Handles/Raises" --> A6 35 | A4 -- "Injects via @pass_context" --> A0 36 | ``` 37 | 38 | -------------------------------------------------------------------------------- /docs/Codex/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Codex" 4 | nav_order: 5 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: Codex 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | Codex[View Repo](https://github.com/openai/codex) is a command-line interface (CLI) tool that functions as an **AI coding assistant**. 13 | It runs in your terminal, allowing you to chat with an AI model (like *GPT-4o*) to understand, modify, and generate code within your projects. 14 | The tool can read files, apply changes (*patches*), and execute shell commands, prioritizing safety through user **approval policies** and command **sandboxing**. It supports both interactive chat and a non-interactive *single-pass mode* for batch operations. 15 | 16 | ```mermaid 17 | flowchart TD 18 | A0["Agent Loop"] 19 | A1["Terminal UI (Ink Components)"] 20 | A2["Approval Policy & Security"] 21 | A3["Command Execution & Sandboxing"] 22 | A4["Configuration Management"] 23 | A5["Response & Tool Call Handling"] 24 | A6["Single-Pass Mode"] 25 | A7["Input Handling (TextBuffer/Editor)"] 26 | A0 -- "Drives updates for" --> A1 27 | A0 -- "Processes responses via" --> A5 28 | A0 -- "Consults policy from" --> A2 29 | A0 -- "Loads config using" --> A4 30 | A1 -- "Uses editor for input" --> A7 31 | A2 -- "Dictates sandboxing for" --> A3 32 | A4 -- "Provides settings to" --> A2 33 | A5 -- "Triggers" --> A3 34 | A7 -- "Provides user input to" --> A0 35 | A0 -- "Can initiate" --> A6 36 | A6 -- "Renders via specific UI" --> A1 37 | ``` -------------------------------------------------------------------------------- /docs/Crawl4AI/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Crawl4AI" 4 | nav_order: 7 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: Crawl4AI 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | `Crawl4AI`[View Repo](https://github.com/unclecode/crawl4ai/tree/9c58e4ce2ee025debd3f36bf213330bd72b90e46/crawl4ai) is a flexible Python library for *asynchronously crawling websites* and *extracting structured content*, specifically designed for **AI use cases**. 13 | You primarily interact with the `AsyncWebCrawler`, which acts as the main coordinator. You provide it with URLs and a `CrawlerRunConfig` detailing *how* to crawl (e.g., using specific strategies for fetching, scraping, filtering, and extraction). 14 | It can handle single pages or multiple URLs concurrently using a `BaseDispatcher`, optionally crawl deeper by following links via `DeepCrawlStrategy`, manage `CacheMode`, and apply `RelevantContentFilter` before finally returning a `CrawlResult` containing all the gathered data. 15 | 16 | ```mermaid 17 | flowchart TD 18 | A0["AsyncWebCrawler"] 19 | A1["CrawlerRunConfig"] 20 | A2["AsyncCrawlerStrategy"] 21 | A3["ContentScrapingStrategy"] 22 | A4["ExtractionStrategy"] 23 | A5["CrawlResult"] 24 | A6["BaseDispatcher"] 25 | A7["DeepCrawlStrategy"] 26 | A8["CacheContext / CacheMode"] 27 | A9["RelevantContentFilter"] 28 | A0 -- "Configured by" --> A1 29 | A0 -- "Uses Fetching Strategy" --> A2 30 | A0 -- "Uses Scraping Strategy" --> A3 31 | A0 -- "Uses Extraction Strategy" --> A4 32 | A0 -- "Produces" --> A5 33 | A0 -- "Uses Dispatcher for `arun_m..." --> A6 34 | A0 -- "Uses Caching Logic" --> A8 35 | A6 -- "Calls Crawler's `arun`" --> A0 36 | A1 -- "Specifies Deep Crawl Strategy" --> A7 37 | A7 -- "Processes Links from" --> A5 38 | A3 -- "Provides Cleaned HTML to" --> A9 39 | A1 -- "Specifies Content Filter" --> A9 40 | ``` -------------------------------------------------------------------------------- /docs/CrewAI/01_crew.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Crew" 4 | parent: "CrewAI" 5 | nav_order: 1 6 | --- 7 | 8 | # Chapter 1: Crew - Your AI Team Manager 9 | 10 | Welcome to the world of CrewAI! We're excited to help you build teams of AI agents that can work together to accomplish complex tasks. 11 | 12 | Imagine you have a big project, like planning a surprise birthday trip for a friend. Doing it all yourself – researching destinations, checking flight prices, finding hotels, planning activities – can be overwhelming. Wouldn't it be great if you had a team to help? Maybe one person researches cool spots, another finds the best travel deals, and you coordinate everything. 13 | 14 | That's exactly what a `Crew` does in CrewAI! It acts like the **project manager** or even the **entire team** itself, bringing together specialized AI assistants ([Agents](02_agent.md)) and telling them what [Tasks](03_task.md) to do and in what order. 15 | 16 | **What Problem Does `Crew` Solve?** 17 | 18 | Single AI models are powerful, but complex goals often require multiple steps and different kinds of expertise. A `Crew` allows you to break down a big goal into smaller, manageable [Tasks](03_task.md) and assign each task to the best AI [Agent](02_agent.md) for the job. It then manages how these agents work together to achieve the overall objective. 19 | 20 | ## What is a Crew? 21 | 22 | Think of a `Crew` as the central coordinator. It holds everything together: 23 | 24 | 1. **The Team ([Agents](02_agent.md)):** It knows which AI agents are part of the team. Each agent might have a specific role (like 'Travel Researcher' or 'Booking Specialist'). 25 | 2. **The Plan ([Tasks](03_task.md)):** It holds the list of tasks that need to be completed to achieve the final goal (e.g., 'Research European cities', 'Find affordable flights', 'Book hotel'). 26 | 3. **The Workflow ([Process](05_process.md)):** It defines *how* the team works. Should they complete tasks one after another (`sequential`)? Or should there be a manager agent delegating work (`hierarchical`)? 27 | 4. **Collaboration:** It orchestrates how agents share information and pass results from one task to the next. 28 | 29 | ## Let's Build a Simple Crew! 30 | 31 | Let's try building a very basic `Crew` for our trip planning example. For now, we'll just set up the structure. We'll learn more about creating sophisticated [Agents](02_agent.md) and [Tasks](03_task.md) in the next chapters. 32 | 33 | ```python 34 | # Import necessary classes (we'll learn about these soon!) 35 | from crewai import Agent, Task, Crew, Process 36 | 37 | # Define our agents (don't worry about the details for now) 38 | # Agent 1: The Researcher 39 | researcher = Agent( 40 | role='Travel Researcher', 41 | goal='Find interesting cities in Europe for a birthday trip', 42 | backstory='An expert travel researcher.', 43 | # verbose=True, # Optional: Shows agent's thinking process 44 | allow_delegation=False # This agent doesn't delegate work 45 | # llm=your_llm # We'll cover LLMs later! 46 | ) 47 | 48 | # Agent 2: The Planner 49 | planner = Agent( 50 | role='Activity Planner', 51 | goal='Create a fun 3-day itinerary for the chosen city', 52 | backstory='An experienced activity planner.', 53 | # verbose=True, 54 | allow_delegation=False 55 | # llm=your_llm 56 | ) 57 | ``` 58 | 59 | **Explanation:** 60 | 61 | * We import `Agent`, `Task`, `Crew`, and `Process` from the `crewai` library. 62 | * We create two simple [Agents](02_agent.md). We give them a `role` and a `goal`. Think of these as job titles and descriptions for our AI assistants. (We'll dive deep into Agents in [Chapter 2](02_agent.md)). 63 | 64 | Now, let's define the [Tasks](03_task.md) for these agents: 65 | 66 | ```python 67 | # Define the tasks 68 | task1 = Task( 69 | description='Identify the top 3 European cities suitable for a sunny birthday trip in May.', 70 | expected_output='A list of 3 cities with brief reasons.', 71 | agent=researcher # Assign task1 to the researcher agent 72 | ) 73 | 74 | task2 = Task( 75 | description='Based on the chosen city from task 1, create a 3-day activity plan.', 76 | expected_output='A detailed itinerary for 3 days.', 77 | agent=planner # Assign task2 to the planner agent 78 | ) 79 | ``` 80 | 81 | **Explanation:** 82 | 83 | * We create two [Tasks](03_task.md). Each task has a `description` (what to do) and an `expected_output` (what the result should look like). 84 | * Crucially, we assign each task to an `agent`. `task1` goes to the `researcher`, and `task2` goes to the `planner`. (More on Tasks in [Chapter 3](03_task.md)). 85 | 86 | Finally, let's assemble the `Crew`: 87 | 88 | ```python 89 | # Create the Crew 90 | trip_crew = Crew( 91 | agents=[researcher, planner], 92 | tasks=[task1, task2], 93 | process=Process.sequential # Tasks will run one after another 94 | # verbose=2 # Optional: Sets verbosity level for the crew execution 95 | ) 96 | 97 | # Start the Crew's work! 98 | result = trip_crew.kickoff() 99 | 100 | print("\n\n########################") 101 | print("## Here is the result") 102 | print("########################\n") 103 | print(result) 104 | ``` 105 | 106 | **Explanation:** 107 | 108 | 1. We create an instance of the `Crew` class. 109 | 2. We pass the list of `agents` we defined earlier. 110 | 3. We pass the list of `tasks`. The order in this list matters for the sequential process. 111 | 4. We set the `process` to `Process.sequential`. This means `task1` will be completed first by the `researcher`, and its output will *automatically* be available as context for `task2` when the `planner` starts working. 112 | 5. We call the `kickoff()` method. This is like saying "Okay team, start working!" 113 | 6. The `Crew` manages the execution, ensuring the `researcher` does `task1`, then the `planner` does `task2`. 114 | 7. The `result` will contain the final output from the *last* task (`task2` in this case). 115 | 116 | **Expected Outcome (Conceptual):** 117 | 118 | When you run this (assuming you have underlying AI models configured, which we'll cover in the [LLM chapter](06_llm.md)), the `Crew` will: 119 | 120 | 1. Ask the `researcher` agent to perform `task1`. 121 | 2. The `researcher` will (conceptually) think and produce a list like: "1. Barcelona (Sunny, vibrant) 2. Lisbon (Coastal, historic) 3. Rome (Iconic, warm)". 122 | 3. The `Crew` takes this output and gives it to the `planner` agent along with `task2`. 123 | 4. The `planner` agent uses the city list (and likely picks one, or you'd refine the task) and creates a 3-day itinerary. 124 | 5. The final `result` printed will be the 3-day itinerary generated by the `planner`. 125 | 126 | ## How Does `Crew.kickoff()` Work Inside? 127 | 128 | You don't *need* to know the deep internals to use CrewAI, but understanding the basics helps! When you call `kickoff()`: 129 | 130 | 1. **Input Check:** It checks if you provided any starting inputs (we didn't in this simple example, but you could provide a starting topic or variable). 131 | 2. **Agent & Task Setup:** It makes sure all agents and tasks are ready to go. It ensures agents have the necessary configurations ([LLMs](06_llm.md), [Tools](04_tool.md) - more on these later!). 132 | 3. **Process Execution:** It looks at the chosen `process` (e.g., `sequential`). 133 | * **Sequential:** It runs tasks one by one. The output of task `N` is added to the context for task `N+1`. 134 | * **Hierarchical (Advanced):** If you chose this process, the Crew would use a dedicated 'manager' agent to coordinate the other agents and decide who does what next. We'll stick to sequential for now. 135 | 4. **Task Execution Loop:** 136 | * It picks the next task based on the process. 137 | * It finds the assigned agent for that task. 138 | * It gives the agent the task description and any relevant context (like outputs from previous tasks). 139 | * The agent performs the task using its underlying AI model ([LLM](06_llm.md)). 140 | * The agent returns the result (output) of the task. 141 | * The Crew stores this output. 142 | * Repeat until all tasks are done. 143 | 5. **Final Output:** The `Crew` packages the output from the final task (and potentially outputs from all tasks) and returns it. 144 | 145 | Let's visualize the `sequential` process: 146 | 147 | ```mermaid 148 | sequenceDiagram 149 | participant User 150 | participant MyCrew as Crew 151 | participant ResearcherAgent as Researcher 152 | participant PlannerAgent as Planner 153 | 154 | User->>MyCrew: kickoff() 155 | MyCrew->>ResearcherAgent: Execute Task 1 ("Find cities...") 156 | Note right of ResearcherAgent: Researcher thinks... generates city list. 157 | ResearcherAgent-->>MyCrew: Task 1 Output ("Barcelona, Lisbon, Rome...") 158 | MyCrew->>PlannerAgent: Execute Task 2 ("Create itinerary...") \nwith Task 1 Output as context 159 | Note right of PlannerAgent: Planner thinks... uses city list, creates itinerary. 160 | PlannerAgent-->>MyCrew: Task 2 Output ("Day 1: ..., Day 2: ...") 161 | MyCrew-->>User: Final Result (Task 2 Output) 162 | ``` 163 | 164 | **Code Glimpse (`crew.py` simplified):** 165 | 166 | The `Crew` class itself is defined in `crewai/crew.py`. It takes parameters like `agents`, `tasks`, and `process` when you create it. 167 | 168 | ```python 169 | # Simplified view from crewai/crew.py 170 | class Crew(BaseModel): 171 | tasks: List[Task] = Field(default_factory=list) 172 | agents: List[BaseAgent] = Field(default_factory=list) 173 | process: Process = Field(default=Process.sequential) 174 | # ... other configurations like memory, cache, etc. 175 | 176 | def kickoff(self, inputs: Optional[Dict[str, Any]] = None) -> CrewOutput: 177 | # ... setup steps ... 178 | 179 | # Decides which execution path based on the process 180 | if self.process == Process.sequential: 181 | result = self._run_sequential_process() 182 | elif self.process == Process.hierarchical: 183 | result = self._run_hierarchical_process() 184 | else: 185 | # Handle other processes or errors 186 | raise NotImplementedError(...) 187 | 188 | # ... cleanup and formatting steps ... 189 | return result # Returns a CrewOutput object 190 | 191 | def _run_sequential_process(self) -> CrewOutput: 192 | # Simplified loop logic 193 | task_outputs = [] 194 | for task in self.tasks: 195 | agent = task.agent # Find the agent for this task 196 | context = self._get_context(task, task_outputs) # Get outputs from previous tasks 197 | # Execute the task (sync or async) 198 | output = task.execute_sync(agent=agent, context=context) 199 | task_outputs.append(output) 200 | # ... logging/callbacks ... 201 | return self._create_crew_output(task_outputs) # Package final result 202 | ``` 203 | 204 | This simplified view shows how the `Crew` holds the `agents` and `tasks`, and the `kickoff` method directs traffic based on the chosen `process`, eventually looping through tasks sequentially if `Process.sequential` is selected. 205 | 206 | ## Conclusion 207 | 208 | You've learned about the most fundamental concept in CrewAI: the `Crew`! It's the manager that brings your AI agents together, gives them tasks, and defines how they collaborate to achieve a larger goal. We saw how to define agents and tasks (at a high level) and assemble them into a `Crew` using a `sequential` process. 209 | 210 | But a Crew is nothing without its members! In the next chapter, we'll dive deep into the first core component: the [Agent](02_agent.md). What makes an agent tick? How do you define their roles, goals, and capabilities? Let's find out! 211 | 212 | --- 213 | 214 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/CrewAI/02_agent.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Agent" 4 | parent: "CrewAI" 5 | nav_order: 2 6 | --- 7 | 8 | # Chapter 2: Agent - Your Specialized AI Worker 9 | 10 | In [Chapter 1](01_crew.md), we learned about the `Crew` – the manager that organizes our AI team. But a manager needs a team to manage! That's where `Agent`s come in. 11 | 12 | ## Why Do We Need Agents? 13 | 14 | Imagine our trip planning `Crew` again. The `Crew` knows the overall goal (plan a surprise trip), but it doesn't *do* the research or the planning itself. It needs specialists. 15 | 16 | * One specialist could be excellent at researching travel destinations. 17 | * Another could be fantastic at creating detailed itineraries. 18 | 19 | In CrewAI, these specialists are called **`Agent`s**. Instead of having one super-smart AI try to juggle everything, we create multiple `Agent`s, each with its own focus and expertise. This makes complex tasks more manageable and often leads to better results. 20 | 21 | **Problem Solved:** `Agent`s allow you to break down a large task into smaller pieces and assign each piece to an AI worker specifically designed for it. 22 | 23 | ## What is an Agent? 24 | 25 | Think of an `Agent` as a **dedicated AI worker** on your `Crew`. Each `Agent` has a unique profile that defines who they are and what they do: 26 | 27 | 1. **`role`**: This is the Agent's job title. What function do they perform in the team? Examples: 'Travel Researcher', 'Marketing Analyst', 'Code Reviewer', 'Blog Post Writer'. 28 | 2. **`goal`**: This is the Agent's primary objective. What specific outcome are they trying to achieve within their role? Examples: 'Find the top 3 family-friendly European destinations', 'Analyze competitor website traffic', 'Identify bugs in Python code', 'Draft an engaging blog post about AI'. 29 | 3. **`backstory`**: This is the Agent's personality, skills, and history. It tells the AI *how* to behave and what expertise it possesses. It adds flavour and context. Examples: 'An expert travel agent with 20 years of experience in European travel.', 'A data-driven market analyst known for spotting emerging trends.', 'A meticulous senior software engineer obsessed with code quality.', 'A witty content creator known for simplifying complex topics.' 30 | 4. **`llm`** (Optional): This is the Agent's "brain" – the specific Large Language Model (like GPT-4, Gemini, etc.) it uses to think, communicate, and execute tasks. We'll cover this more in the [LLM chapter](06_llm.md). If not specified, it usually inherits the `Crew`'s default LLM. 31 | 5. **`tools`** (Optional): These are special capabilities the Agent can use, like searching the web, using a calculator, or reading files. Think of them as the Agent's equipment. We'll explore these in the [Tool chapter](04_tool.md). 32 | 6. **`allow_delegation`** (Optional, default `False`): Can this Agent ask other Agents in the `Crew` for help with a sub-task? If `True`, it enables collaboration. 33 | 7. **`verbose`** (Optional, default `False`): If `True`, the Agent will print out its thought process as it works, which is great for debugging and understanding what's happening. 34 | 35 | An Agent takes the [Tasks](03_task.md) assigned to it by the `Crew` and uses its `role`, `goal`, `backstory`, `llm`, and `tools` to complete them. 36 | 37 | ## Let's Define an Agent! 38 | 39 | Let's revisit the `researcher` Agent from Chapter 1 and look closely at how it's defined. 40 | 41 | ```python 42 | # Make sure you have crewai installed 43 | # pip install crewai 44 | 45 | from crewai import Agent 46 | 47 | # Define our researcher agent 48 | researcher = Agent( 49 | role='Expert Travel Researcher', 50 | goal='Find the most exciting and sunny European cities for a birthday trip in late May.', 51 | backstory=( 52 | "You are a world-class travel researcher with deep knowledge of " 53 | "European destinations. You excel at finding hidden gems and understanding " 54 | "weather patterns. Your recommendations are always insightful and tailored." 55 | ), 56 | verbose=True, # We want to see the agent's thinking process 57 | allow_delegation=False # This agent focuses on its own research 58 | # tools=[...] # We'll add tools later! 59 | # llm=your_llm # We'll cover LLMs later! 60 | ) 61 | 62 | # (You would typically define other agents, tasks, and a crew here) 63 | # print(researcher) # Just to see the object 64 | ``` 65 | 66 | **Explanation:** 67 | 68 | * `from crewai import Agent`: We import the necessary `Agent` class. 69 | * `role='Expert Travel Researcher'`: We clearly define the agent's job title. This tells the LLM its primary function. 70 | * `goal='Find the most exciting...'`: We give it a specific, measurable objective. This guides its actions. 71 | * `backstory='You are a world-class...'`: We provide context and personality. This influences the *style* and *quality* of its output. Notice the detailed description – this helps the LLM adopt the persona. 72 | * `verbose=True`: We'll see detailed logs of this agent's thoughts and actions when it runs. 73 | * `allow_delegation=False`: This researcher won't ask other agents for help; it will complete its task independently. 74 | 75 | Running this code snippet creates an `Agent` object in Python. This object is now ready to be added to a [Crew](01_crew.md) and assigned [Tasks](03_task.md). 76 | 77 | ## How Agents Work "Under the Hood" 78 | 79 | So, what happens when an `Agent` is given a task by the `Crew`? 80 | 81 | 1. **Receive Task & Context:** The `Agent` gets the task description (e.g., "Find 3 sunny cities") and potentially some context from previous tasks (e.g., "The user prefers coastal cities"). 82 | 2. **Consult Profile:** It looks at its own `role`, `goal`, and `backstory`. This helps it frame *how* to tackle the task. Our 'Expert Travel Researcher' will approach this differently than a 'Budget Backpacker Blogger'. 83 | 3. **Think & Plan (Using LLM):** The `Agent` uses its assigned `llm` (its brain) to think. It breaks down the task, formulates a plan, and decides what information it needs. This often involves an internal "monologue" (which you can see if `verbose=True`). 84 | 4. **Use Tools (If Necessary):** If the plan requires external information or actions (like searching the web for current weather or calculating travel times), and the agent *has* the right [Tools](04_tool.md), it will use them. 85 | 5. **Delegate (If Allowed & Necessary):** If `allow_delegation=True` and the `Agent` decides a sub-part of the task is better handled by another specialist `Agent` in the `Crew`, it can ask the `Crew` to delegate that part. 86 | 6. **Generate Output (Using LLM):** Based on its thinking, tool results, and potentially delegated results, the `Agent` uses its `llm` again to formulate the final response or output for the task. 87 | 7. **Return Result:** The `Agent` passes its completed work back to the `Crew`. 88 | 89 | Let's visualize this simplified flow: 90 | 91 | ```mermaid 92 | sequenceDiagram 93 | participant C as Crew 94 | participant MyAgent as Agent (Researcher) 95 | participant LLM as Agent's Brain 96 | participant SearchTool as Tool 97 | 98 | C->>MyAgent: Execute Task ("Find sunny cities in May") 99 | MyAgent->>MyAgent: Consult profile (Role, Goal, Backstory) 100 | MyAgent->>LLM: Formulate plan & Ask: "Best way to find sunny cities?" 101 | LLM-->>MyAgent: Suggestion: "Search web for 'Europe weather May'" 102 | MyAgent->>SearchTool: Use Tool(query="Europe weather May sunny cities") 103 | SearchTool-->>MyAgent: Web search results (e.g., Lisbon, Seville, Malta) 104 | MyAgent->>LLM: Consolidate results & Ask: "Format these 3 cities nicely" 105 | LLM-->>MyAgent: Formatted list: "1. Lisbon..." 106 | MyAgent-->>C: Task Result ("Here are 3 sunny cities: Lisbon...") 107 | 108 | ``` 109 | 110 | **Diving into the Code (`agent.py`)** 111 | 112 | The core logic for the `Agent` resides in the `crewai/agent.py` file. 113 | 114 | The `Agent` class itself inherits from `BaseAgent` (`crewai/agents/agent_builder/base_agent.py`) and primarily stores the configuration you provide: 115 | 116 | ```python 117 | # Simplified view from crewai/agent.py 118 | from crewai.agents.agent_builder.base_agent import BaseAgent 119 | # ... other imports 120 | 121 | class Agent(BaseAgent): 122 | role: str = Field(description="Role of the agent") 123 | goal: str = Field(description="Objective of the agent") 124 | backstory: str = Field(description="Backstory of the agent") 125 | llm: Any = Field(default=None, description="LLM instance") 126 | tools: Optional[List[BaseTool]] = Field(default_factory=list) 127 | allow_delegation: bool = Field(default=False) 128 | verbose: bool = Field(default=False) 129 | # ... other fields like memory, max_iter, etc. 130 | 131 | def execute_task( 132 | self, 133 | task: Task, 134 | context: Optional[str] = None, 135 | tools: Optional[List[BaseTool]] = None, 136 | ) -> str: 137 | # ... (steps 1 & 2: Prepare task prompt with context, memory, knowledge) ... 138 | 139 | task_prompt = task.prompt() # Get base task description 140 | if context: 141 | task_prompt = f"{task_prompt}\nContext:\n{context}" 142 | # Add memory, knowledge, tool descriptions etc. to the prompt... 143 | 144 | # ... (Internal setup: Create AgentExecutor if needed) ... 145 | self.create_agent_executor(tools=tools or self.tools) 146 | 147 | # ... (Step 3-7: Run the execution loop via AgentExecutor) ... 148 | result = self.agent_executor.invoke({ 149 | "input": task_prompt, 150 | "tool_names": self._get_tool_names(self.agent_executor.tools), 151 | "tools": self._get_tool_descriptions(self.agent_executor.tools), 152 | # ... other inputs for the executor ... 153 | })["output"] # Extract the final string output 154 | 155 | return result 156 | 157 | def create_agent_executor(self, tools: Optional[List[BaseTool]] = None) -> None: 158 | # Sets up the internal CrewAgentExecutor which handles the actual 159 | # interaction loop with the LLM and tools. 160 | # It uses the agent's profile (role, goal, backstory) to build the main prompt. 161 | pass 162 | 163 | # ... other helper methods ... 164 | ``` 165 | 166 | Key takeaways from the code: 167 | 168 | * The `Agent` class mainly holds the configuration (`role`, `goal`, `backstory`, `llm`, `tools`, etc.). 169 | * The `execute_task` method is called by the `Crew` when it's the agent's turn. 170 | * It prepares a detailed prompt for the underlying LLM, incorporating the task, context, the agent's profile, and available tools. 171 | * It uses an internal object called `agent_executor` (specifically `CrewAgentExecutor` from `crewai/agents/crew_agent_executor.py`) to manage the actual step-by-step thinking, tool use, and response generation loop with the LLM. 172 | 173 | You don't need to understand the `agent_executor` in detail right now, just know that it's the engine that drives the agent's execution based on the profile and task you provide. 174 | 175 | ## Conclusion 176 | 177 | You've now met the core members of your AI team: the `Agent`s! You learned that each `Agent` is a specialized worker defined by its `role`, `goal`, and `backstory`. They use an [LLM](06_llm.md) as their brain and can be equipped with [Tools](04_tool.md) to perform specific actions. 178 | 179 | We saw how to define an agent in code and got a glimpse into how they process information and execute the work assigned by the [Crew](01_crew.md). 180 | 181 | But defining an `Agent` is only half the story. What specific work should they *do*? How do we describe the individual steps needed to achieve the `Crew`'s overall objective? That's where the next concept comes in: the [Task](03_task.md). Let's dive into defining the actual work! 182 | 183 | --- 184 | 185 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/CrewAI/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "CrewAI" 4 | nav_order: 8 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: CrewAI 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | **CrewAI**[View Repo](https://github.com/crewAIInc/crewAI/tree/e723e5ca3fb7e4cb890c4befda47746aedbd7408/src/crewai) is a framework for orchestrating *autonomous AI agents*. 13 | Think of it like building a specialized team (a **Crew**) where each member (**Agent**) has a role, goal, and tools. 14 | You assign **Tasks** to Agents, defining what needs to be done. The **Crew** manages how these Agents collaborate, following a specific **Process** (like sequential steps). 15 | Agents use their "brain" (an **LLM**) and can utilize **Tools** (like web search) and access shared **Memory** or external **Knowledge** bases to complete their tasks effectively. 16 | 17 | ```mermaid 18 | flowchart TD 19 | A0["Agent"] 20 | A1["Task"] 21 | A2["Crew"] 22 | A3["Tool"] 23 | A4["Process"] 24 | A5["LLM"] 25 | A6["Memory"] 26 | A7["Knowledge"] 27 | A2 -- "Manages" --> A0 28 | A2 -- "Orchestrates" --> A1 29 | A2 -- "Defines workflow" --> A4 30 | A2 -- "Manages shared" --> A6 31 | A0 -- "Executes" --> A1 32 | A0 -- "Uses" --> A3 33 | A0 -- "Uses as brain" --> A5 34 | A0 -- "Queries" --> A7 35 | A1 -- "Assigned to" --> A0 36 | ``` -------------------------------------------------------------------------------- /docs/DSPy/02_signature.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Signature" 4 | parent: "DSPy" 5 | nav_order: 2 6 | --- 7 | 8 | # Chapter 2: Signatures - Defining the Task 9 | 10 | In [Chapter 1: Modules and Programs](01_module___program.md), we learned that `Module`s are like Lego bricks that perform specific tasks, often using Language Models ([LM](05_lm__language_model_client_.md)). We saw how `Program`s combine these modules. 11 | 12 | But how does a `Module`, especially one using an LM like `dspy.Predict`, know *exactly* what job to do? 13 | 14 | Imagine you ask a chef (our LM) to cook something. Just saying "cook" isn't enough! You need to tell them: 15 | 1. **What ingredients to use** (the inputs). 16 | 2. **What dish to make** (the outputs). 17 | 3. **The recipe or instructions** (how to make it). 18 | 19 | This is precisely what a **`Signature`** does in DSPy! 20 | 21 | A `Signature` acts like a clear recipe or contract for a DSPy `Module`. It defines: 22 | 23 | * **Input Fields:** What information the module needs to start its work. 24 | * **Output Fields:** What information the module is expected to produce. 25 | * **Instructions:** Natural language guidance (like a recipe!) telling the underlying LM *how* to transform the inputs into the outputs. 26 | 27 | Think of it as specifying the 'shape' and 'purpose' of a module, making sure everyone (you, DSPy, and the LM) understands the task. 28 | 29 | ## Why Do We Need Signatures? 30 | 31 | Without a clear definition, how would a module like `dspy.Predict` know what to ask the LM? 32 | 33 | Let's say we want a module to translate English text to French. We need to tell it: 34 | * It needs an `english_sentence` as input. 35 | * It should produce a `french_sentence` as output. 36 | * The *task* is to translate the input sentence into French. 37 | 38 | A `Signature` bundles all this information together neatly. 39 | 40 | ## Defining a Signature: The Recipe Card 41 | 42 | The most common way to define a Signature is by creating a Python class that inherits from `dspy.Signature`. 43 | 44 | Let's create our English-to-French translation signature: 45 | 46 | ```python 47 | import dspy 48 | from dspy.signatures.field import InputField, OutputField 49 | 50 | class TranslateToFrench(dspy.Signature): 51 | """Translates English text to French.""" # <-- These are the Instructions! 52 | 53 | # Define the Input Field the module expects 54 | english_sentence = dspy.InputField(desc="The original sentence in English") 55 | 56 | # Define the Output Field the module should produce 57 | french_sentence = dspy.OutputField(desc="The translated sentence in French") 58 | 59 | ``` 60 | 61 | Let's break this down: 62 | 63 | 1. **`class TranslateToFrench(dspy.Signature):`**: We declare a new class named `TranslateToFrench` that inherits from `dspy.Signature`. This tells DSPy it's a signature definition. 64 | 2. **`"""Translates English text to French."""`**: This is the **docstring**. It's crucial! DSPy uses this docstring as the natural language **Instructions** for the LM. It tells the LM the *goal* of the task. 65 | 3. **`english_sentence = dspy.InputField(...)`**: We define an input field named `english_sentence`. `dspy.InputField` marks this as required input. The `desc` provides a helpful description (good for documentation and potentially useful for the LM later). 66 | 4. **`french_sentence = dspy.OutputField(...)`**: We define an output field named `french_sentence`. `dspy.OutputField` marks this as the expected output. The `desc` describes what this field should contain. 67 | 68 | That's it! We've created a reusable "recipe card" that clearly defines our translation task. 69 | 70 | ## How Modules Use Signatures 71 | 72 | Now, how does a `Module` like `dspy.Predict` use this `TranslateToFrench` signature? 73 | 74 | `dspy.Predict` is a pre-built module designed to take a signature and use an LM to generate the output fields based on the input fields and instructions. 75 | 76 | Here's how you might use our signature with `dspy.Predict` (we'll cover `dspy.Predict` in detail in [Chapter 4](04_predict.md)): 77 | 78 | ```python 79 | # Assume 'lm' is a configured Language Model client (more in Chapter 5) 80 | # lm = dspy.OpenAI(model='gpt-3.5-turbo') 81 | # dspy.settings.configure(lm=lm) 82 | 83 | # Create an instance of dspy.Predict, giving it our Signature 84 | translator = dspy.Predict(TranslateToFrench) 85 | 86 | # Call the predictor with the required input field 87 | english = "Hello, how are you?" 88 | result = translator(english_sentence=english) 89 | 90 | # The result object will contain the output field defined in the signature 91 | print(f"English: {english}") 92 | # Assuming the LM works correctly, it might print: 93 | # print(f"French: {result.french_sentence}") # => French: Bonjour, comment ça va? 94 | ``` 95 | 96 | In this (slightly simplified) example: 97 | 98 | 1. `translator = dspy.Predict(TranslateToFrench)`: We create a `Predict` module. Crucially, we pass our `TranslateToFrench` **class** itself to it. `dspy.Predict` now knows the input/output fields and the instructions from the signature. 99 | 2. `result = translator(english_sentence=english)`: When we call the `translator`, we provide the input data using the exact name defined in our signature (`english_sentence`). 100 | 3. `result.french_sentence`: `dspy.Predict` uses the LM, guided by the signature's instructions and fields, to generate the output. It then returns an object where you can access the generated French text using the output field name (`french_sentence`). 101 | 102 | The `Signature` acts as the bridge, ensuring the `Predict` module knows its job specification. 103 | 104 | ## How It Works Under the Hood (A Peek) 105 | 106 | You don't need to memorize this, but understanding the flow helps! When a module like `dspy.Predict` uses a `Signature`: 107 | 108 | 1. **Inspection:** The module looks at the `Signature` class (`TranslateToFrench` in our case). 109 | 2. **Extract Info:** It identifies the `InputField`s (`english_sentence`), `OutputField`s (`french_sentence`), and the `Instructions` (the docstring: `"Translates English text to French."`). 110 | 3. **Prompt Formatting:** When you call the module (e.g., `translator(english_sentence="Hello")`), it uses this information to build a prompt for the [LM](05_lm__language_model_client_.md). This prompt typically includes: 111 | * The **Instructions**. 112 | * Clearly labeled **Input Fields** and their values. 113 | * Clearly labeled **Output Fields** (often just the names, indicating what the LM should generate). 114 | 4. **LM Call:** The formatted prompt is sent to the configured LM. 115 | 5. **Parsing Output:** The LM's response is received. DSPy tries to parse this response to extract the values for the defined `OutputField`s (like `french_sentence`). 116 | 6. **Return Result:** A structured result object containing the parsed outputs is returned. 117 | 118 | Let's visualize this flow: 119 | 120 | ```mermaid 121 | sequenceDiagram 122 | participant User 123 | participant PredictModule as dspy.Predict(TranslateToFrench) 124 | participant Signature as TranslateToFrench 125 | participant LM as Language Model 126 | 127 | User->>PredictModule: Call with english_sentence="Hello" 128 | PredictModule->>Signature: Get Instructions, Input/Output Fields 129 | Signature-->>PredictModule: Return structure ("Translates...", "english_sentence", "french_sentence") 130 | PredictModule->>LM: Send formatted prompt (e.g., "Translate...\nEnglish: Hello\nFrench:") 131 | LM-->>PredictModule: Return generated text (e.g., "Bonjour") 132 | PredictModule->>Signature: Parse LM output into 'french_sentence' field 133 | Signature-->>PredictModule: Return structured output {french_sentence: "Bonjour"} 134 | PredictModule-->>User: Return structured output (Prediction object) 135 | ``` 136 | 137 | The core logic for defining signatures resides in: 138 | 139 | * `dspy/signatures/signature.py`: Defines the base `Signature` class and the logic for handling instructions and fields. 140 | * `dspy/signatures/field.py`: Defines `InputField` and `OutputField`. 141 | 142 | Modules like `dspy.Predict` (in `dspy/predict/predict.py`) contain the code to *read* these Signatures and interact with LMs accordingly. 143 | 144 | ```python 145 | # Simplified view inside dspy/signatures/signature.py 146 | from pydantic import BaseModel 147 | from pydantic.fields import FieldInfo 148 | # ... other imports ... 149 | 150 | class SignatureMeta(type(BaseModel)): 151 | # Metaclass magic to handle fields and docstring 152 | def __new__(mcs, name, bases, namespace, **kwargs): 153 | # ... logic to find fields, handle docstring ... 154 | cls = super().__new__(mcs, name, bases, namespace, **kwargs) 155 | cls.__doc__ = cls.__doc__ or _default_instructions(cls) # Default instructions if none provided 156 | # ... logic to validate fields ... 157 | return cls 158 | 159 | @property 160 | def instructions(cls) -> str: 161 | # Retrieves the docstring as instructions 162 | return inspect.cleandoc(getattr(cls, "__doc__", "")) 163 | 164 | @property 165 | def input_fields(cls) -> dict[str, FieldInfo]: 166 | # Finds fields marked as input 167 | return cls._get_fields_with_type("input") 168 | 169 | @property 170 | def output_fields(cls) -> dict[str, FieldInfo]: 171 | # Finds fields marked as output 172 | return cls._get_fields_with_type("output") 173 | 174 | class Signature(BaseModel, metaclass=SignatureMeta): 175 | # The base class you inherit from 176 | pass 177 | 178 | # Simplified view inside dspy/signatures/field.py 179 | import pydantic 180 | 181 | def InputField(**kwargs): 182 | # Creates a Pydantic field marked as input for DSPy 183 | return pydantic.Field(**move_kwargs(**kwargs, __dspy_field_type="input")) 184 | 185 | def OutputField(**kwargs): 186 | # Creates a Pydantic field marked as output for DSPy 187 | return pydantic.Field(**move_kwargs(**kwargs, __dspy_field_type="output")) 188 | 189 | ``` 190 | 191 | The key takeaway is that the `Signature` class structure (using `InputField`, `OutputField`, and the docstring) provides a standardized way for modules to understand the task specification. 192 | 193 | ## Conclusion 194 | 195 | You've now learned about `Signatures`, the essential component for defining *what* a DSPy module should do! 196 | 197 | * A `Signature` specifies the **Inputs**, **Outputs**, and **Instructions** for a task. 198 | * It acts like a contract or recipe card for modules, especially those using LMs. 199 | * You typically define them by subclassing `dspy.Signature`, using `InputField`, `OutputField`, and a descriptive **docstring** for instructions. 200 | * Modules like `dspy.Predict` use Signatures to understand the task and generate appropriate prompts for the LM. 201 | 202 | Signatures bring clarity and structure to LM interactions. But how do we provide concrete examples to help the LM learn or perform better? That's where `Examples` come in! 203 | 204 | **Next:** [Chapter 3: Example](03_example.md) 205 | 206 | --- 207 | 208 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/DSPy/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "DSPy" 4 | nav_order: 9 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: DSPy 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | DSPy[View Repo](https://github.com/stanfordnlp/dspy/tree/7cdfe988e6404289b896d946d957f17bb4d9129b/dspy) helps you build and optimize *programs* that use **Language Models (LMs)** and **Retrieval Models (RMs)**. 13 | Think of it like composing Lego bricks (**Modules**) where each brick performs a specific task (like generating text or retrieving information). 14 | **Signatures** define what each Module does (its inputs and outputs), and **Teleprompters** automatically tune these modules (like optimizing prompts or examples) to get the best performance on your data. 15 | 16 | ```mermaid 17 | flowchart TD 18 | A0["Module / Program"] 19 | A1["Signature"] 20 | A2["Predict"] 21 | A3["LM (Language Model Client)"] 22 | A4["RM (Retrieval Model Client)"] 23 | A5["Teleprompter / Optimizer"] 24 | A6["Example"] 25 | A7["Evaluate"] 26 | A8["Adapter"] 27 | A9["Settings"] 28 | A0 -- "Contains / Composes" --> A0 29 | A0 -- "Uses (via Retrieve)" --> A4 30 | A1 -- "Defines structure for" --> A6 31 | A2 -- "Implements" --> A1 32 | A2 -- "Calls" --> A3 33 | A2 -- "Uses demos from" --> A6 34 | A2 -- "Formats prompts using" --> A8 35 | A5 -- "Optimizes" --> A0 36 | A5 -- "Fine-tunes" --> A3 37 | A5 -- "Uses training data from" --> A6 38 | A5 -- "Uses metric from" --> A7 39 | A7 -- "Tests" --> A0 40 | A7 -- "Evaluates on dataset of" --> A6 41 | A8 -- "Translates" --> A1 42 | A8 -- "Formats demos from" --> A6 43 | A9 -- "Configures default" --> A3 44 | A9 -- "Configures default" --> A4 45 | A9 -- "Configures default" --> A8 46 | ``` -------------------------------------------------------------------------------- /docs/FastAPI/08_background_tasks.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Background Tasks" 4 | parent: "FastAPI" 5 | nav_order: 8 6 | --- 7 | 8 | # Chapter 8: Background Tasks 9 | 10 | Welcome back! In [Chapter 7: Security Utilities](07_security_utilities.md), we learned how to protect our API endpoints using FastAPI's security features. Now, let's explore how to perform actions *after* we've already sent a response back to the user. 11 | 12 | ## What Problem Does This Solve? 13 | 14 | Imagine a user registers on your website. When they submit their registration form, your API endpoint needs to: 15 | 16 | 1. Create the new user account in the database. 17 | 2. Send a welcome email to the user. 18 | 3. Send a notification to an admin. 19 | 4. Return a "Success!" message to the user. 20 | 21 | Creating the user (step 1) is quick and essential before confirming success. But sending emails or notifications (steps 2 and 3) can sometimes be slow. Should the user have to wait several extra seconds just for the emails to be sent before they see the "Success!" message? Probably not! It would be much better if the API could send the "Success!" response immediately after creating the user, and then handle sending the emails *in the background*. 22 | 23 | This is exactly what **Background Tasks** allow you to do in FastAPI. They let you define operations that need to happen *after* the response has been sent to the client, ensuring your users get a fast response time for the main action. 24 | 25 | **Analogy:** Think of your path operation function as having a conversation with the user (sending the response). Once the main conversation is finished, you might hand off a follow-up task (like mailing a letter) to an assistant to complete later, so you don't keep the user waiting. Background Tasks are like that helpful assistant. 26 | 27 | ## Key Concepts 28 | 29 | 1. **`BackgroundTasks` Object:** A special object provided by FastAPI that holds a list of tasks to be run later. 30 | 2. **Dependency Injection:** You get access to this object by declaring it as a parameter in your path operation function, just like we learned in [Chapter 5: Dependency Injection](05_dependency_injection.md). Example: `def my_endpoint(background_tasks: BackgroundTasks): ...`. 31 | 3. **`add_task()` Method:** You use the `add_task()` method on the `BackgroundTasks` object to schedule a function to run in the background. You provide the function itself and any arguments it needs. Example: `background_tasks.add_task(send_welcome_email, user.email, user.name)`. 32 | 4. **Post-Response Execution:** FastAPI (specifically, the underlying Starlette framework) ensures that all functions added via `add_task()` are executed *only after* the response has been successfully sent back to the client. 33 | 34 | ## Using Background Tasks 35 | 36 | Let's create a simple example. Imagine we want to write a message to a log file *after* sending a notification response to the user. 37 | 38 | **Step 1: Import `BackgroundTasks`** 39 | 40 | First, import the necessary class from `fastapi`. 41 | 42 | ```python 43 | # main.py (or your router file) 44 | from fastapi import BackgroundTasks, FastAPI 45 | 46 | app = FastAPI() 47 | ``` 48 | 49 | **Step 2: Define the Task Function** 50 | 51 | This is the function you want to run in the background. It can be a regular `def` function or an `async def` function. 52 | 53 | ```python 54 | # A function to simulate writing to a log 55 | # In a real app, this might send an email, process data, etc. 56 | def write_log(message: str): 57 | # Simulate writing to a file 58 | with open("log.txt", mode="a") as log_file: 59 | log_file.write(message + "\n") 60 | print(f"Log written: {message}") # Also print to console for demo 61 | 62 | ``` 63 | 64 | **Explanation:** 65 | * This is a simple Python function `write_log` that takes a `message` string. 66 | * It opens a file named `log.txt` in "append" mode (`a`) and writes the message to it. 67 | * We also print to the console so we can easily see when it runs during testing. 68 | 69 | **Step 3: Inject `BackgroundTasks` and use `add_task`** 70 | 71 | Now, modify your path operation function to accept `BackgroundTasks` as a parameter and use its `add_task` method. 72 | 73 | ```python 74 | @app.post("/send-notification/{email}") 75 | async def send_notification( 76 | email: str, 77 | background_tasks: BackgroundTasks # Inject BackgroundTasks 78 | ): 79 | # The message we want to log in the background 80 | log_message = f"Notification sent to: {email}" 81 | 82 | # Add the task to run after the response 83 | background_tasks.add_task(write_log, log_message) # Schedule write_log 84 | 85 | # Return the response immediately 86 | return {"message": "Notification sent successfully!"} 87 | 88 | ``` 89 | 90 | **Explanation:** 91 | 92 | * `background_tasks: BackgroundTasks`: We declare a parameter named `background_tasks` with the type hint `BackgroundTasks`. FastAPI's dependency injection system will automatically create and provide a `BackgroundTasks` object here. 93 | * `background_tasks.add_task(write_log, log_message)`: This is the crucial line. 94 | * We call the `add_task` method on the injected `background_tasks` object. 95 | * The first argument is the function we want to run in the background (`write_log`). 96 | * The subsequent arguments (`log_message`) are the arguments that will be passed to our `write_log` function when it's eventually called. 97 | * `return {"message": "Notification sent successfully!"}`: The function returns its response *without* waiting for `write_log` to finish. 98 | 99 | **How it Behaves:** 100 | 101 | 1. **Run the App:** `uvicorn main:app --reload` 102 | 2. **Send a Request:** Use `curl` or the `/docs` UI to send a `POST` request to `/send-notification/test@example.com`. 103 | ```bash 104 | curl -X POST http://127.0.0.1:8000/send-notification/test@example.com 105 | ``` 106 | 3. **Immediate Response:** You will immediately receive the JSON response: 107 | ```json 108 | {"message":"Notification sent successfully!"} 109 | ``` 110 | 4. **Background Execution:** *After* the response above has been sent, look at your Uvicorn console output. You will see the message: 111 | ``` 112 | Log written: Notification sent to: test@example.com 113 | ``` 114 | Also, check your project directory. A file named `log.txt` will have been created (or appended to) with the content: 115 | ``` 116 | Notification sent to: test@example.com 117 | ``` 118 | 119 | This demonstrates that the `write_log` function ran *after* the client received the success message, preventing any delay for the user. 120 | 121 | ## How it Works Under the Hood (Simplified) 122 | 123 | What's happening behind the scenes when you use `BackgroundTasks`? 124 | 125 | 1. **Request In:** A request arrives at your FastAPI application (e.g., `POST /send-notification/test@example.com`). 126 | 2. **Dependency Injection:** FastAPI processes the request, routes it to `send_notification`, and prepares its dependencies. It sees the `background_tasks: BackgroundTasks` parameter and creates an empty `BackgroundTasks` object instance. 127 | 3. **Path Function Runs:** Your `send_notification` function is called with the `email` and the empty `background_tasks` object. 128 | 4. **`add_task` Called:** Your code calls `background_tasks.add_task(write_log, log_message)`. This doesn't *run* `write_log` yet; it just adds the function (`write_log`) and its arguments (`log_message`) to an internal list within the `background_tasks` object. 129 | 5. **Response Returned:** Your path function finishes and returns the dictionary `{"message": "Notification sent successfully!"}`. 130 | 6. **Middleware Magic (Starlette):** FastAPI (using Starlette middleware) takes the response object *and* the `background_tasks` object (which now contains the scheduled task). 131 | 7. **Response Sent:** The middleware sends the HTTP response (`200 OK` with the JSON body) back to the client over the network. 132 | 8. **Tasks Executed:** *After* the response has been sent, the Starlette middleware iterates through the tasks stored in the `background_tasks` object. For each task, it calls the stored function (`write_log`) with the stored arguments (`log_message`). This happens in the server's process, separate from the initial request-response flow. 133 | 134 | Here's a simplified sequence diagram: 135 | 136 | ```mermaid 137 | sequenceDiagram 138 | participant Client 139 | participant FastAPIApp as FastAPI App (via Starlette) 140 | participant PathFunc as send_notification 141 | participant BGTasks as BackgroundTasks Object 142 | participant BGExecutor as Background Task Executor (Starlette) 143 | participant TaskFunc as write_log 144 | 145 | Client->>+FastAPIApp: POST /send-notification/test@example.com 146 | FastAPIApp->>FastAPIApp: Route to send_notification 147 | FastAPIApp->>+PathFunc: Call send_notification(email="...", background_tasks=BGTasks) 148 | PathFunc->>+BGTasks: background_tasks.add_task(write_log, "...") 149 | BGTasks-->>-PathFunc: Task added to internal list 150 | PathFunc-->>-FastAPIApp: Return response {"message": "..."} 151 | Note over FastAPIApp: FastAPI/Starlette prepares to send response AND notes background tasks 152 | FastAPIApp-->>-Client: Send HTTP 200 OK Response 153 | Note over FastAPIApp: Response sent, now run background tasks 154 | FastAPIApp->>+BGExecutor: Execute tasks from BGTasks object 155 | BGExecutor->>+TaskFunc: Call write_log("...") 156 | TaskFunc->>TaskFunc: Write to log.txt 157 | TaskFunc-->>-BGExecutor: Task finished 158 | BGExecutor-->>-FastAPIApp: All tasks finished 159 | ``` 160 | 161 | ### Code Connections 162 | 163 | * **`fastapi.BackgroundTasks`**: This class (in `fastapi/background.py`) inherits directly from `starlette.background.BackgroundTasks`. It mostly just provides type hints and documentation specific to FastAPI. 164 | * **`BackgroundTasks.add_task`**: This method simply calls the `add_task` method of the parent Starlette class. 165 | * **`starlette.background.BackgroundTasks`**: This is where the core logic resides (in the `starlette` library, which FastAPI builds upon). It stores tasks as tuples of `(callable, args, kwargs)`. 166 | * **`starlette.middleware.exceptions.ExceptionMiddleware` (and potentially others):** Starlette's middleware stack, particularly around exception handling and response sending, is responsible for checking if a `BackgroundTasks` object exists on the response object after the main endpoint code has run. If tasks exist, the middleware ensures they are executed *after* the response is sent using `anyio.create_task_group().start_soon()` or similar mechanisms. See `starlette.responses.Response.__call__`. 167 | 168 | Essentially, FastAPI provides a convenient way (via dependency injection) to access Starlette's background task functionality. 169 | 170 | ## Conclusion 171 | 172 | You've learned how to use FastAPI's `BackgroundTasks` to perform operations *after* sending a response to the client! 173 | 174 | * You understand that this is useful for **slow or non-critical tasks** (like sending emails or notifications) that shouldn't delay the user's primary action. 175 | * You learned to inject the **`BackgroundTasks`** object as a dependency. 176 | * You saw how to schedule functions using the **`add_task(func, *args, **kwargs)`** method. 177 | * You understand that these tasks run **after the response** has been delivered. 178 | 179 | This feature helps you build more responsive APIs by deferring non-essential work. 180 | 181 | This chapter concludes our core introduction to FastAPI! We've covered setting up applications, defining routes, handling parameters and data validation, using dependency injection, handling errors, securing endpoints, and now running background tasks. With these building blocks, you can create powerful and efficient web APIs. 182 | 183 | Where do you go from here? You can dive deeper into the official FastAPI documentation to explore advanced topics like WebSockets, middleware, bigger application structures, testing, and deployment. Happy coding! 184 | 185 | --- 186 | 187 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/FastAPI/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "FastAPI" 4 | nav_order: 10 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: FastAPI 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | FastAPI[View Repo](https://github.com/fastapi/fastapi/tree/628c34e0cae200564d191c95d7edea78c88c4b5e/fastapi) is a modern, *high-performance* web framework for building APIs with Python. 13 | It's designed to be **easy to use**, fast to code, and ready for production. 14 | Key features include **automatic data validation** (using Pydantic), **dependency injection**, and **automatic interactive API documentation** (OpenAPI and Swagger UI). 15 | 16 | ```mermaid 17 | flowchart TD 18 | A0["FastAPI Application & Routing"] 19 | A1["Path Operations & Parameter Declaration"] 20 | A2["Data Validation & Serialization (Pydantic)"] 21 | A3["Dependency Injection"] 22 | A4["OpenAPI & Automatic Docs"] 23 | A5["Error Handling"] 24 | A6["Security Utilities"] 25 | A7["Background Tasks"] 26 | A0 -- "Defines Routes for" --> A1 27 | A1 -- "Uses for parameter/body val..." --> A2 28 | A1 -- "Uses Depends() for dependen..." --> A3 29 | A0 -- "Generates API spec for" --> A4 30 | A0 -- "Manages global" --> A5 31 | A3 -- "Injects BackgroundTasks object" --> A7 32 | A6 -- "Uses Depends mechanism (Sec..." --> A3 33 | A6 -- "Raises HTTPException on fai..." --> A5 34 | A4 -- "Reads definitions from" --> A1 35 | A4 -- "Reads Pydantic models for s..." --> A2 36 | A4 -- "Reads security scheme defin..." --> A6 37 | A5 -- "Handles RequestValidationEr..." --> A2 38 | ``` -------------------------------------------------------------------------------- /docs/Flask/01_application_object___flask__.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Application Object (Flask)" 4 | parent: "Flask" 5 | nav_order: 1 6 | --- 7 | 8 | # Chapter 1: Application Object (`Flask`) 9 | 10 | Welcome to your first step into the world of Flask! Flask is a "microframework" for building web applications in Python. "Micro" doesn't mean it's limited; it means Flask provides the essentials to get started quickly, letting you add features as needed. 11 | 12 | In this chapter, we'll explore the absolute heart of any Flask application: the **Application Object**. 13 | 14 | ## What Problem Does It Solve? The Need for a Control Tower 15 | 16 | Imagine you're building a simple website. Maybe it just needs to show "Hello, World!" when someone visits the homepage. How does the web server know *what* Python code to run when a request comes in for `/` (the homepage)? How does it manage different pages (like `/about` or `/contact`)? How does it handle settings or connect to other tools? 17 | 18 | You need a central place to manage all these tasks. Think of a busy airport: you need a **control tower** to direct planes (incoming web requests), manage runways (URL paths), and coordinate ground crew (other parts of your application). 19 | 20 | In Flask, the `Flask` object is that control tower. It's the main object you create that represents your entire web application. 21 | 22 | ## Creating Your First Flask Application 23 | 24 | Let's create the simplest possible Flask app. You'll need a Python file (let's call it `hello.py`). 25 | 26 | 1. **Import Flask:** First, you need to bring the `Flask` class into your code. 27 | 2. **Create an Instance:** Then, you create an *instance* of this class. This instance *is* your application. 28 | 29 | ```python 30 | # hello.py 31 | 32 | from flask import Flask 33 | 34 | # Create the application object 35 | app = Flask(__name__) 36 | 37 | # We'll add more here soon! 38 | ``` 39 | 40 | Let's break down `app = Flask(__name__)`: 41 | 42 | * `from flask import Flask`: This line imports the necessary `Flask` class from the Flask library you installed. 43 | * `app = Flask(...)`: This creates the actual application object. We usually call the variable `app`, but you could name it something else. 44 | * `__name__`: This is a special Python variable. When you run a Python script directly, Python sets `__name__` to the string `"__main__"`. If the script is imported by another script, `__name__` is set to the module's name (e.g., `"hello"` if your file is `hello.py`). 45 | * **Why `__name__`?** Flask uses this argument to figure out the *location* of your application. This helps it find other files like templates and static assets (images, CSS) later on. For simple, single-module applications, using `__name__` is standard practice and almost always correct. The Flask documentation notes that if you're building a larger application structured as a Python package, you might hardcode the package name instead (like `app = Flask('yourapplication')`), but for beginners, `__name__` is the way to go. 46 | 47 | This `app` object is now ready to be configured and run. 48 | 49 | ## Adding a Basic Route 50 | 51 | Our `app` object doesn't do anything yet. Let's tell it what to do when someone visits the homepage (`/`). We do this using a *route*. We'll cover routing in detail in the next chapter, but here's a taste: 52 | 53 | ```python 54 | # hello.py (continued) 55 | 56 | from flask import Flask 57 | 58 | app = Flask(__name__) 59 | 60 | # Define what happens when someone visits the homepage ("/") 61 | @app.route('/') 62 | def index(): 63 | return 'Hello, World!' 64 | 65 | # More code to run the app below... 66 | ``` 67 | 68 | * `@app.route('/')`: This is a Python decorator. It modifies the function defined right below it (`index`). It tells our `app` object: "When a web request comes in for the URL path `/`, call the `index` function." 69 | * `def index(): ...`: This is a simple Python function. Flask calls these "view functions." 70 | * `return 'Hello, World!'`: Whatever the view function returns is sent back to the user's web browser as the response. 71 | 72 | ## Running Your Application 73 | 74 | How do we start the web server so people can actually visit our page? We use the `app` object's `run()` method. It's common practice to put this inside a special `if` block: 75 | 76 | ```python 77 | # hello.py (end of the file) 78 | 79 | from flask import Flask 80 | 81 | app = Flask(__name__) 82 | 83 | @app.route('/') 84 | def index(): 85 | return 'Hello, World!' 86 | 87 | # This block runs the app only when the script is executed directly 88 | if __name__ == '__main__': 89 | # Start the built-in development server 90 | app.run(debug=True) 91 | ``` 92 | 93 | * `if __name__ == '__main__':`: This standard Python construct ensures that the code inside it only runs when you execute `hello.py` directly (like typing `python hello.py` in your terminal). It prevents the server from starting if you were to *import* `hello.py` into another Python file. 94 | * `app.run()`: This method starts Flask's built-in development web server. This server is great for testing but **not** suitable for production (live websites). 95 | * `debug=True`: This enables Flask's "debug mode". It provides helpful error messages in the browser and automatically restarts the server whenever you save changes to your code, making development much easier. **Never use debug mode in production!** 96 | 97 | **To run this:** 98 | 99 | 1. Save the complete code as `hello.py`. 100 | 2. Open your terminal or command prompt. 101 | 3. Navigate to the directory where you saved the file. 102 | 4. Run the command: `python hello.py` 103 | 5. You'll see output like this: 104 | ``` 105 | * Serving Flask app 'hello' 106 | * Debug mode: on 107 | * Running on http://127.0.0.1:5000 (Press CTRL+C to quit) 108 | * Restarting with stat 109 | * Debugger is active! 110 | * Debugger PIN: ... 111 | ``` 112 | 6. Open your web browser and go to `http://127.0.0.1:5000/`. 113 | 7. You should see the text "Hello, World!" 114 | 115 | You've just created and run your first Flask application! The `app = Flask(__name__)` line was the crucial first step, creating the central object that manages everything. 116 | 117 | ## Under the Hood: What Happens When You Create `Flask(__name__)`? 118 | 119 | While you don't *need* to know the deep internals right away, a little insight helps understanding. When you call `app = Flask(__name__)`, several things happen inside Flask (simplified): 120 | 121 | 1. **Initialization:** The `Flask` class's `__init__` method (found in `app.py`, inheriting from `App` in `sansio/app.py`) is called. 122 | 2. **Path Determination:** It uses the `import_name` (`__name__`) you passed to figure out the application's `root_path`. This is like finding the main hangar at the airport. (See `get_root_path` in `helpers.py` and `find_package` in `sansio/scaffold.py`). 123 | 3. **Configuration Setup:** It creates a configuration object (`self.config`), usually an instance of the `Config` class (from `config.py`). This object holds settings like `DEBUG`, `SECRET_KEY`, etc. We'll cover this in [Configuration (`Config`)](06_configuration___config__.md). 124 | 4. **URL Map Creation:** It creates a `URL Map` (`self.url_map`), which is responsible for matching incoming request URLs to your view functions. This is core to the [Routing System](02_routing_system.md). 125 | 5. **Internal Structures:** It sets up various internal dictionaries to store things like your view functions (`self.view_functions`), error handlers (`self.error_handler_spec`), functions to run before/after requests, etc. 126 | 6. **Static Route (Optional):** If you configured a `static_folder` (Flask does by default), it automatically adds a URL rule (like `/static/`) to serve static files like CSS and JavaScript. 127 | 128 | Here's a simplified diagram of the process: 129 | 130 | ```mermaid 131 | sequenceDiagram 132 | participant UserCode as hello.py 133 | participant Flask as Flask(__init__) 134 | participant App as Base App(__init__) 135 | participant Config as Config() 136 | participant URLMap as URL Map() 137 | 138 | UserCode->>+Flask: app = Flask(__name__) 139 | Flask->>+App: Initialize base features (paths, folders) 140 | App-->>-Flask: Base initialized 141 | Flask->>+Config: Create config object (self.config) 142 | Config-->>-Flask: Config ready 143 | Flask->>+URLMap: Create URL map (self.url_map) 144 | URLMap-->>-Flask: Map ready 145 | Flask-->>-UserCode: Return Flask instance (app) 146 | ``` 147 | 148 | The `app` object returned is now the fully initialized "control tower," ready to register routes and handle requests. 149 | 150 | ## Conclusion 151 | 152 | You've learned about the most fundamental concept in Flask: the **Application Object**, created by instantiating the `Flask` class (usually as `app = Flask(__name__)`). This object acts as the central registry and controller for your entire web application. It's where you define URL routes, manage configuration, and connect various components. 153 | 154 | We saw how to create a minimal application, add a simple route using `@app.route()`, and run the development server using `app.run()`. 155 | 156 | Now that you have your central `app` object, the next logical step is to understand how Flask directs incoming web requests to the correct Python functions. That's the job of the routing system. 157 | 158 | Ready to direct some traffic? Let's move on to [Routing System](02_routing_system.md). 159 | 160 | --- 161 | 162 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/Flask/02_routing_system.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Routing System" 4 | parent: "Flask" 5 | nav_order: 2 6 | --- 7 | 8 | # Chapter 2: Routing System 9 | 10 | Welcome back! In [Chapter 1: Application Object (`Flask`)](01_application_object___flask__.md), we learned how to create the central `app` object, the control tower for our Flask application. We even added a simple "Hello, World!" page using `@app.route('/')`. 11 | 12 | But how did Flask know that visiting the homepage (`/`) should run our `index()` function? And how can we create more pages, like an "About Us" page at `/about`? That's where the **Routing System** comes in. 13 | 14 | ## What Problem Does It Solve? The Need for Directions 15 | 16 | Imagine you have a website with multiple pages: a homepage, an about page, a contact page, maybe even pages for individual user profiles. When a user types a URL like `http://yourwebsite.com/about` into their browser, how does your Flask application know *which* piece of Python code should handle this request and generate the "About Us" content? 17 | 18 | You need a system to map these incoming URLs to the specific Python functions that generate the response for each page. Think of it like a city map's index: 19 | 20 | * **URL:** The street address you want to find (e.g., `/about`). 21 | * **Routing System:** The index in the map book. 22 | * **View Function:** The specific page number in the map book that shows the details for that address. 23 | 24 | Flask's routing system, largely powered by a library called Werkzeug, acts as this index. It lets you define URL patterns (like `/` or `/about` or `/user/`) and connect them to your Python functions (called **view functions**). 25 | 26 | ## Defining Routes with `@app.route()` 27 | 28 | In Flask, the most common way to define these URL-to-function mappings is using the `@app.route()` decorator, which we briefly saw in Chapter 1. 29 | 30 | Let's revisit our `hello.py` and add an "About" page. 31 | 32 | 1. We keep the route for the homepage (`/`). 33 | 2. We add a *new* route for `/about`. 34 | 35 | ```python 36 | # hello.py 37 | 38 | from flask import Flask 39 | 40 | # Create the application object from Chapter 1 41 | app = Flask(__name__) 42 | 43 | # Route for the homepage 44 | @app.route('/') 45 | def index(): 46 | return 'Welcome to the Homepage!' 47 | 48 | # NEW: Route for the about page 49 | @app.route('/about') 50 | def about(): 51 | return 'This is the About Us page.' 52 | 53 | # Code to run the app (from Chapter 1) 54 | if __name__ == '__main__': 55 | app.run(debug=True) 56 | ``` 57 | 58 | **Explanation:** 59 | 60 | * `@app.route('/')`: This tells Flask: "If a request comes in for the URL path `/`, execute the function directly below (`index`)." 61 | * `@app.route('/about')`: This tells Flask: "If a request comes in for the URL path `/about`, execute the function directly below (`about`)." 62 | * `def index(): ...` and `def about(): ...`: These are our **view functions**. They contain the Python code that runs for their respective routes and must return the response to send back to the browser. 63 | 64 | **Running this:** 65 | 66 | 1. Save the code as `hello.py`. 67 | 2. Run `python hello.py` in your terminal. 68 | 3. Visit `http://127.0.0.1:5000/` in your browser. You should see "Welcome to the Homepage!". 69 | 4. Visit `http://127.0.0.1:5000/about`. You should see "This is the About Us page.". 70 | 71 | See? The routing system directed each URL to the correct view function! 72 | 73 | ## Dynamic Routes: Using Variables in URLs 74 | 75 | What if you want pages that change based on the URL? For example, a profile page for different users like `/user/alice` and `/user/bob`. You don't want to write a new view function for every single user! 76 | 77 | Flask allows you to define *variable parts* in your URL rules using angle brackets `< >`. 78 | 79 | Let's create a dynamic route to greet users: 80 | 81 | ```python 82 | # hello.py (continued) 83 | 84 | # ... (keep Flask import, app creation, index, and about routes) ... 85 | 86 | # NEW: Dynamic route for user profiles 87 | @app.route('/user/') 88 | def show_user_profile(username): 89 | # The 'username' variable from the URL is passed to the function! 90 | return f'Hello, {username}!' 91 | 92 | # ... (keep the if __name__ == '__main__': block) ... 93 | ``` 94 | 95 | **Explanation:** 96 | 97 | * `@app.route('/user/')`: 98 | * The `/user/` part is fixed. 99 | * `` is a **variable placeholder**. Flask will match any text here (like `alice`, `bob`, `123`) and capture it. 100 | * `def show_user_profile(username):`: 101 | * Notice the function now accepts an argument named `username`. This **must match** the variable name used in the angle brackets in the route. 102 | * Flask automatically passes the value captured from the URL to this argument. 103 | * `return f'Hello, {username}!'`: We use an f-string to include the captured username in the response. 104 | 105 | **Running this:** 106 | 107 | 1. Save the updated `hello.py` (make sure `debug=True` is still set so the server restarts). 108 | 2. Visit `http://127.0.0.1:5000/user/Alice`. You should see "Hello, Alice!". 109 | 3. Visit `http://127.0.0.1:5000/user/Bob`. You should see "Hello, Bob!". 110 | 111 | Flask's routing system matched both URLs to the same rule (`/user/`) and passed the different usernames (`'Alice'`, `'Bob'`) to the `show_user_profile` function. 112 | 113 | ## Specifying Data Types: Converters 114 | 115 | By default, variables captured from the URL are treated as strings. But what if you need a number? For example, displaying blog post number 5 at `/post/5`. You might want Flask to ensure that only numbers are accepted for that part of the URL. 116 | 117 | You can specify a **converter** inside the angle brackets using ``. 118 | 119 | Let's add a route for blog posts using the `int` converter: 120 | 121 | ```python 122 | # hello.py (continued) 123 | 124 | # ... (keep previous code) ... 125 | 126 | # NEW: Route for displaying a specific blog post by ID 127 | @app.route('/post/') 128 | def show_post(post_id): 129 | # Flask ensures post_id is an integer and passes it here 130 | # Note: We are just showing the ID, not actually fetching a post 131 | return f'Showing Post Number: {post_id} (Type: {type(post_id).__name__})' 132 | 133 | # ... (keep the if __name__ == '__main__': block) ... 134 | ``` 135 | 136 | **Explanation:** 137 | 138 | * `@app.route('/post/')`: 139 | * `` tells Flask: "Match this part of the URL, but only if it looks like an integer. Convert it to an integer and pass it as the `post_id` variable." 140 | * `def show_post(post_id):`: The `post_id` argument will now receive an actual Python `int`. 141 | 142 | **Running this:** 143 | 144 | 1. Save the updated `hello.py`. 145 | 2. Visit `http://127.0.0.1:5000/post/123`. You should see "Showing Post Number: 123 (Type: int)". 146 | 3. Visit `http://127.0.0.1:5000/post/abc`. You'll get a "Not Found" error! Why? Because `abc` doesn't match the `int` converter, so Flask doesn't consider this URL to match the rule. 147 | 148 | Common converters include: 149 | 150 | * `string`: (Default) Accepts any text without a slash. 151 | * `int`: Accepts positive integers. 152 | * `float`: Accepts positive floating-point values. 153 | * `path`: Like `string` but also accepts slashes (useful for matching file paths). 154 | * `uuid`: Accepts UUID strings. 155 | 156 | ## Under the Hood: How Does Routing Work? 157 | 158 | You don't *need* to know the deep internals, but understanding the basics helps. 159 | 160 | When you define routes using `@app.route()`, Flask doesn't immediately check URLs. Instead, it builds a map, like pre-compiling that map index we talked about. 161 | 162 | 1. **Building the Map:** 163 | * When you create your `app = Flask(__name__)` ([Chapter 1](01_application_object___flask__.md)), Flask initializes an empty `URLMap` object (from the Werkzeug library, stored in `app.url_map`). See `Flask.__init__` in `app.py` which calls `super().__init__` in `sansio/app.py`, which creates the `self.url_map`. 164 | * Each time you use `@app.route('/some/rule', ...)` or directly call `app.add_url_rule(...)` (see `sansio/scaffold.py`), Flask creates a `Rule` object (like `Rule('/user/')`) describing the pattern, the allowed HTTP methods (GET, POST, etc.), the endpoint name (usually the function name), and any converters. 165 | * This `Rule` object is added to the `app.url_map`. 166 | 167 | 2. **Matching a Request:** 168 | * When a request like `GET /user/Alice` arrives, Flask's `wsgi_app` method (in `app.py`) gets called. 169 | * It uses the `app.url_map` and the incoming request environment (URL path, HTTP method) to find a matching `Rule`. Werkzeug's `MapAdapter.match()` method (created via `app.create_url_adapter` which calls `url_map.bind_to_environ`) does the heavy lifting here. 170 | * If a match is found for `/user/`, `match()` returns the endpoint name (e.g., `'show_user_profile'`) and a dictionary of the extracted variables (e.g., `{'username': 'Alice'}`). These get stored on the `request` object ([Chapter 3](03_request_and_response_objects.md)) as `request.url_rule` and `request.view_args`. 171 | * If no rule matches, a "Not Found" (404) error is raised. 172 | 173 | 3. **Dispatching to the View Function:** 174 | * Flask's `app.dispatch_request()` method (in `app.py`) takes the endpoint name from `request.url_rule.endpoint`. 175 | * It looks up the actual Python view function associated with that endpoint name in the `app.view_functions` dictionary (which `@app.route` also populated). 176 | * It calls the view function, passing the extracted variables from `request.view_args` as keyword arguments (e.g., `show_user_profile(username='Alice')`). 177 | * The return value of the view function becomes the response. 178 | 179 | Here's a simplified diagram of the matching process: 180 | 181 | ```mermaid 182 | sequenceDiagram 183 | participant Browser 184 | participant FlaskApp as app.wsgi_app 185 | participant URLMap as url_map.bind(...).match() 186 | participant ViewFunc as show_user_profile() 187 | 188 | Browser->>+FlaskApp: GET /user/Alice 189 | FlaskApp->>+URLMap: Match path '/user/Alice' and method 'GET'? 190 | URLMap-->>-FlaskApp: Match found! Endpoint='show_user_profile', Args={'username': 'Alice'} 191 | FlaskApp->>+ViewFunc: Call show_user_profile(username='Alice') 192 | ViewFunc-->>-FlaskApp: Return 'Hello, Alice!' 193 | FlaskApp-->>-Browser: Send response 'Hello, Alice!' 194 | ``` 195 | 196 | The key takeaway is that `@app.route` builds a map upfront, and Werkzeug efficiently searches this map for each incoming request to find the right function and extract any variable parts. 197 | 198 | ## Conclusion 199 | 200 | You've learned how Flask's **Routing System** acts as a map between URLs and the Python functions (view functions) that handle them. 201 | 202 | * We use the `@app.route()` decorator to define URL rules. 203 | * We can create static routes (like `/about`) and dynamic routes using variables (`/user/`). 204 | * Converters (``) allow us to specify the expected data type for URL variables, providing automatic validation and conversion. 205 | * Under the hood, Flask and Werkzeug build a `URLMap` from these rules and use it to efficiently dispatch incoming requests to the correct view function. 206 | 207 | Now that we know how to direct requests to the right functions, what information comes *with* a request (like form data or query parameters)? And how do we properly format the data we send *back*? That's where the Request and Response objects come in. 208 | 209 | Let's dive into [Chapter 3: Request and Response Objects](03_request_and_response_objects.md). 210 | 211 | --- 212 | 213 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/Flask/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Flask" 4 | nav_order: 11 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: Flask 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | Flask[View Repo](https://github.com/pallets/flask/tree/ab8149664182b662453a563161aa89013c806dc9/src/flask) is a lightweight **web framework** for Python. 13 | It helps you build web applications by handling incoming *web requests* and sending back *responses*. 14 | Flask provides tools for **routing** URLs to your Python functions, managing *request data*, creating *responses*, and using *templates* to generate HTML. 15 | 16 | ```mermaid 17 | flowchart TD 18 | A0["0: Application Object (Flask)"] 19 | A1["1: Blueprints"] 20 | A2["2: Routing System"] 21 | A3["3: Request and Response Objects"] 22 | A4["4: Application and Request Contexts"] 23 | A5["5: Context Globals (current_app, request, session, g)"] 24 | A6["6: Configuration (Config)"] 25 | A7["7: Templating (Jinja2 Integration)"] 26 | A0 -- "Registers" --> A1 27 | A0 -- "Uses" --> A2 28 | A0 -- "Handles" --> A3 29 | A0 -- "Manages" --> A4 30 | A0 -- "Holds" --> A6 31 | A0 -- "Integrates" --> A7 32 | A1 -- "Defines routes using" --> A2 33 | A2 -- "Matches URL from" --> A3 34 | A3 -- "Bound within" --> A4 35 | A4 -- "Enables access to" --> A5 36 | A7 -- "Accesses" --> A5 37 | ``` -------------------------------------------------------------------------------- /docs/Google A2A/01_agent_card.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Agent Card" 4 | parent: "Google A2A" 5 | nav_order: 1 6 | --- 7 | 8 | # Chapter 1: Agent Card - The AI's Business Card 9 | 10 | Welcome to the Google Agent-to-Agent (A2A) Protocol tutorial! Imagine a world full of helpful AI assistants, or "agents." Maybe one agent is great at translating languages, another excels at summarizing long documents, and a third can book appointments. How do these agents, potentially built by different companies using different technologies, find each other and figure out how to work together? 11 | 12 | That's where the **Agent Card** comes in. It solves the problem of **discovery** – how one agent or application can learn about another agent's existence, capabilities, and how to communicate with it. 13 | 14 | Think of it like this: 15 | 16 | * **You want to hire a plumber.** How do you find one? You might look them up online, find their website, or get their business card. This tells you their name, what services they offer (fixing leaks, installing pipes), and how to contact them (phone number, address). 17 | * **An application (or another agent) wants to use an AI agent.** How does it find one? It looks for the agent's **Agent Card**. 18 | 19 | ## What is an Agent Card? 20 | 21 | An **Agent Card** is a small, standardized file, usually named `agent.json`, that acts like a public profile or digital business card for an AI agent. It's typically hosted by the agent itself at a predictable web address. 22 | 23 | This card contains essential information: 24 | 25 | 1. **Who is the agent?** (Name, description, version, who made it) 26 | 2. **What can it do?** (List of skills, like "translate_text" or "summarize_document") 27 | 3. **How do I talk to it?** (The agent's web address/URL, what kind of inputs it understands - text, files, structured data?) 28 | 4. **Does it have special features?** (Like supporting real-time updates via streaming?) 29 | 30 | By reading this card, other agents or applications can quickly understand if this agent is the right one for a job and exactly how to start a conversation (or, in technical terms, initiate a [Task](02_task.md)). 31 | 32 | ## Finding and Reading the Card (Discovery) 33 | 34 | Just like many websites have a standard `robots.txt` file to tell search engines what to do, A2A agents typically make their Agent Card available at a standard path: `/.well-known/agent.json`. 35 | 36 | So, if an agent lives at `http://my-translator-agent.com`, its Agent Card would likely be found at `http://my-translator-agent.com/.well-known/agent.json`. 37 | 38 | Let's see how a client application might fetch this card using Python. 39 | 40 | ```python 41 | # File: demo/ui/utils/agent_card.py (simplified) 42 | import requests # A library to make web requests 43 | from common.types import AgentCard # A helper to understand the card's structure 44 | 45 | def get_agent_card(remote_agent_address: str) -> AgentCard: 46 | """Gets the agent card from the agent's address.""" 47 | agent_card_url = f"{remote_agent_address}/.well-known/agent.json" 48 | print(f"Fetching card from: {agent_card_url}") 49 | # Make a web request to get the file 50 | response = requests.get(agent_card_url) 51 | response.raise_for_status() # Check if the request was successful 52 | # Parse the JSON file content into an AgentCard object 53 | return AgentCard(**response.json()) 54 | 55 | # Example Usage: 56 | agent_address = "http://example-agent.com" # Assume our agent is here 57 | try: 58 | card = get_agent_card(agent_address) 59 | print(f"Got card for agent: {card.name}") 60 | except requests.exceptions.RequestException as e: 61 | print(f"Could not fetch card: {e}") 62 | ``` 63 | 64 | **Explanation:** 65 | 66 | 1. We define the `agent_address` where the agent lives. 67 | 2. The function builds the full URL to the standard `agent.json` path. 68 | 3. It uses the `requests` library to make an HTTP GET request, just like your web browser does when you visit a page. 69 | 4. If the request is successful (HTTP status 200 OK), it takes the JSON text returned by the server and parses it into a structured `AgentCard` object that the program can easily use. 70 | 71 | ### Example `agent.json` 72 | 73 | Here's a simplified example of what the `agent.json` file might look like: 74 | 75 | ```json 76 | // File: /.well-known/agent.json (Example) 77 | { 78 | "name": "Text Summarizer Bot", 79 | "description": "Summarizes long text documents.", 80 | "version": "1.0.0", 81 | "url": "http://example-agent.com/a2a", // Where to send tasks 82 | "capabilities": { 83 | "streaming": false // Doesn't support real-time updates 84 | }, 85 | "defaultInputModes": ["text"], // Primarily accepts text 86 | "defaultOutputModes": ["text"], // Primarily outputs text 87 | "skills": [ 88 | { 89 | "id": "summarize", 90 | "name": "Summarize Text", 91 | "description": "Provide text, get a short summary." 92 | } 93 | ], 94 | "provider": { 95 | "organization": "AI Helpers Inc." 96 | } 97 | } 98 | ``` 99 | 100 | **Explanation:** 101 | 102 | * `name`, `description`, `version`, `provider`: Basic identification info. 103 | * `url`: The specific endpoint *within* the agent's server where A2A communication happens (we'll use this later when sending a [Task](02_task.md)). 104 | * `capabilities`: Tells us if it supports advanced features like `streaming`. This one doesn't. 105 | * `defaultInputModes`/`defaultOutputModes`: What kind of data it generally works with (here, just plain `text`). 106 | * `skills`: A list of specific things this agent can do. This one has a "summarize" skill. 107 | 108 | ## Under the Hood: The Discovery Flow 109 | 110 | How does fetching the Agent Card actually work between the client and the agent (server)? It's a simple web request: 111 | 112 | ```mermaid 113 | sequenceDiagram 114 | participant C as Client App 115 | participant A as Agent Server 116 | C->>A: GET /.well-known/agent.json 117 | Note right of A: Agent looks for its agent.json file 118 | A-->>C: 200 OK (Returns content of agent.json) 119 | Note left of C: Client parses the JSON data 120 | ``` 121 | 122 | **Steps:** 123 | 124 | 1. **Client Request:** The client application (e.g., our Python script) sends an HTTP GET request to the agent's base URL + `/.well-known/agent.json`. 125 | 2. **Server Response:** The agent's server receives the request, finds its `agent.json` file, and sends its content back to the client with a success status (like `200 OK`). 126 | 3. **Client Processing:** The client receives the JSON data and processes it to understand the agent's capabilities. 127 | 128 | The provided sample code includes helper classes to make this easier: 129 | 130 | * **Python:** The `A2ACardResolver` class (`samples/python/common/client/card_resolver.py`) handles fetching and parsing the card. 131 | * **JavaScript:** The `cli.ts` sample (`samples/js/src/cli.ts`) uses the standard `fetch` API to get the card directly. 132 | 133 | ```typescript 134 | // File: samples/js/src/cli.ts (Relevant Snippet) 135 | async function fetchAndDisplayAgentCard() { 136 | const wellKnownUrl = new URL("/.well-known/agent.json", serverUrl).toString(); 137 | console.log(`Attempting to fetch agent card from: ${wellKnownUrl}`); 138 | try { 139 | // Use browser's fetch to get the card 140 | const response = await fetch(wellKnownUrl); 141 | if (response.ok) { 142 | const card: AgentCard = await response.json(); // Parse JSON 143 | agentName = card.name || "Agent"; 144 | console.log(`✓ Agent Card Found: ${agentName}`); 145 | // ... display other card info ... 146 | } else { 147 | console.log(`⚠️ Could not fetch agent card (Status: ${response.status})`); 148 | } 149 | } catch (error: any) { 150 | console.log(`⚠️ Error fetching agent card: ${error.message}`); 151 | } 152 | } 153 | ``` 154 | 155 | This JavaScript code does essentially the same thing as the Python example: builds the URL, fetches the content, and parses the JSON if successful. 156 | 157 | ## Conclusion 158 | 159 | The Agent Card is the cornerstone of discovery in the A2A protocol. It's the agent's public announcement, telling the world who it is, what it can do, and how to interact with it. By fetching and reading this simple `agent.json` file, clients can dynamically discover and prepare to communicate with diverse AI agents. 160 | 161 | Now that we understand how to *find* an agent and learn its basic properties using the Agent Card, we need to learn how to actually *give it work* to do. This brings us to the concept of a **Task**. 162 | 163 | Ready to learn how to ask an agent to perform an action? Let's move on to the next chapter! 164 | 165 | **Next:** [Chapter 2: Task](02_task.md) 166 | 167 | --- 168 | 169 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/Google A2A/03_a2a_protocol___core_types.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "A2A Protocol & Core Types" 4 | parent: "Google A2A" 5 | nav_order: 3 6 | --- 7 | 8 | # Chapter 3: A2A Protocol & Core Types 9 | 10 | In the previous chapters, we learned how to find an agent using its [Agent Card](01_agent_card.md) and how to give it work using a [Task](02_task.md). Think of it like finding a specific workshop (Agent Card) and submitting a work order (Task). 11 | 12 | But how do the client (who submits the order) and the agent (the workshop) actually *talk* to each other? What language do they use? If the client writes the order in English, but the workshop only understands Spanish, nothing will get done! 13 | 14 | This chapter tackles that problem: **How do different AI agents, possibly built by different teams using different technologies, communicate reliably?** 15 | 16 | The answer lies in the **A2A Protocol** and its **Core Types**. 17 | 18 | ## What is a Protocol? The Rules of the Road 19 | 20 | Imagine trying to drive in a country where you don't know the traffic rules. Do you drive on the left or right? What do the signs mean? It would be chaos! Traffic rules are a **protocol** – a shared set of rules everyone agrees on so things run smoothly. 21 | 22 | Similarly, the **A2A Protocol** is the set of rules for how AI agents communicate. It defines: 23 | 24 | 1. **The Transport:** *How* messages physically travel (usually over the internet using standard HTTP requests, like your web browser uses). 25 | 2. **The Format:** *What* the messages look like (the structure and language used). 26 | 3. **The Actions:** *What* commands one agent can send to another (like "start a task" or "cancel a task"). 27 | 28 | Think of it as the **shared language** for AI agents. Just like humans use languages like English or Spanish, which have grammar (rules) and vocabulary (words), the A2A protocol provides the grammar and vocabulary for agents. 29 | 30 | ## The Grammar: JSON-RPC 2.0 31 | 32 | For the A2A protocol, the chosen "grammar" is a standard called **JSON-RPC 2.0**. Don't let the name scare you! It's just a simple way to structure messages using JSON (JavaScript Object Notation - a very common text format for data). 33 | 34 | Here's the basic idea: 35 | 36 | * **Client sends a Request:** The client wanting the agent to do something sends a `Request` message. 37 | * **Agent sends a Response:** The agent replies with a `Response` message. 38 | 39 | A typical JSON-RPC Request looks like this: 40 | 41 | ```json 42 | { 43 | "jsonrpc": "2.0", // Specifies the protocol version 44 | "method": "some_action", // What the client wants the agent to DO 45 | "params": { ... }, // The details needed for the action 46 | "id": "request-123" // A unique ID to match request and response 47 | } 48 | ``` 49 | 50 | **Explanation:** 51 | 52 | * `jsonrpc`: Always "2.0". 53 | * `method`: The name of the command or function the client wants the agent to run (like `tasks/send` from Chapter 2). 54 | * `params`: The input data needed for that command (like the text to translate). This can be an object `{}` or a list `[]`. 55 | * `id`: A unique identifier the client makes up. 56 | 57 | The agent then processes this request and sends back a Response matching that `id`: 58 | 59 | **Success Response:** 60 | 61 | ```json 62 | { 63 | "jsonrpc": "2.0", 64 | "result": { ... }, // The output/result of the action 65 | "id": "request-123" // The SAME ID as the request 66 | } 67 | ``` 68 | 69 | **Error Response:** 70 | 71 | ```json 72 | { 73 | "jsonrpc": "2.0", 74 | "error": { // Details about what went wrong 75 | "code": -32601, 76 | "message": "Method not found" 77 | }, 78 | "id": "request-123" // The SAME ID as the request (or null if error was severe) 79 | } 80 | ``` 81 | 82 | **Explanation:** 83 | 84 | * If the action worked, the response includes a `result` field containing the output. 85 | * If something went wrong, it includes an `error` field with a numeric `code` and a descriptive `message`. 86 | * Crucially, the `id` matches the request, so the client knows which request this response belongs to. 87 | 88 | ```mermaid 89 | sequenceDiagram 90 | participant C as Client App 91 | participant A as Agent Server 92 | 93 | C->>A: JSON-RPC Request (id: "req-abc", method: "tasks/send", params: {...}) 94 | Note right of A: Agent parses JSON, finds method 'tasks/send' 95 | 96 | alt Action Successful 97 | A-->>C: JSON-RPC Response (id: "req-abc", result: {Task Object}) 98 | else Action Failed 99 | A-->>C: JSON-RPC Response (id: "req-abc", error: {code:..., message:...}) 100 | end 101 | Note left of C: Client matches response 'id' to original request 102 | ``` 103 | 104 | This simple request/response structure using JSON-RPC is the foundation of how A2A agents talk. 105 | 106 | ## The Vocabulary: Core Data Types 107 | 108 | If JSON-RPC is the grammar, then the **Core Types** are the standard vocabulary – the specific kinds of "words" or data structures used within the `params` and `result` fields. We've already seen some of these! 109 | 110 | Let's recap the most important ones: 111 | 112 | * **`AgentCard`**: ([Chapter 1](01_agent_card.md)) The agent's profile. Describes its name, skills, and communication endpoint (`url`). Found in `/.well-known/agent.json`. 113 | * Defined in: `samples/js/src/schema.ts:AgentCard`, `samples/python/common/types.py:AgentCard` 114 | 115 | * **`Task`**: ([Chapter 2](02_task.md)) The work order. Contains the unique `id`, current `status`, final `artifacts` (results), etc. 116 | * Defined in: `samples/python/common/types.py:Task`, `samples/js/src/schema.ts:Task` 117 | 118 | * **`Message`**: Represents one turn in the conversation (either from the `user` or the `agent`). Contains one or more `Parts`. 119 | * Defined in: `samples/python/common/types.py:Message`, `samples/js/src/schema.ts:Message` 120 | 121 | * **`Part`**: The actual content within a `Message` or `Artifact`. This is how we send different kinds of data: 122 | * `TextPart`: For plain text. 123 | * `FilePart`: For files (either included directly as encoded text (`bytes`) or as a link (`uri`)). 124 | * `DataPart`: For structured JSON data (like filling out a form). 125 | * Defined in: `samples/python/common/types.py:Part`, `samples/js/src/schema.ts:Part` 126 | 127 | * **`Artifact`**: Represents an output generated by the agent during a `Task`. It also contains `Parts`. For example, if a Task was "create a presentation about cats", an Artifact might be a `FilePart` containing the presentation file. 128 | * Defined in: `samples/python/common/types.py:Artifact`, `samples/js/src/schema.ts:Artifact` 129 | 130 | * **`TaskStatus`**: Holds the current progress state of a `Task`. Includes the `state` itself and a `timestamp`. 131 | * Defined in: `samples/python/common/types.py:TaskStatus`, `samples/js/src/schema.ts:TaskStatus` 132 | 133 | * **`TaskState`**: The specific state within `TaskStatus`. Common values are: `submitted`, `working`, `completed`, `failed`, `canceled`. 134 | * Defined in: `samples/python/common/types.py:TaskState`, `samples/js/src/schema.ts:TaskState` 135 | 136 | **Example: Building a `Message`** 137 | 138 | Let's say the user wants to send the text "Translate 'hello' to French". This would be structured as a `Message` containing a `TextPart`: 139 | 140 | ```json 141 | // This structure would go inside the "params" of a tasks/send request 142 | { 143 | "role": "user", // Who is sending this message 144 | "parts": [ // List of content parts (here, just one) 145 | { 146 | "type": "text", // Specifies this is a TextPart 147 | "text": "Translate 'hello' to French" 148 | } 149 | ] 150 | } 151 | ``` 152 | 153 | If the user also wanted to attach a document for translation, the `parts` list would have two items: a `TextPart` with instructions and a `FilePart` with the document. 154 | 155 | ## Putting It Together: The `tasks/send` Example 156 | 157 | Remember the `tasks/send` request from Chapter 2? Let's look at the full JSON-RPC structure that the client sends over HTTP: 158 | 159 | ```json 160 | // Client Sends This (HTTP POST body to Agent's URL) 161 | { 162 | "jsonrpc": "2.0", 163 | "method": "tasks/send", // The action: start/continue a task 164 | "params": { // The details (TaskSendParams structure) 165 | "id": "task-xyz-789", // Unique Task ID 166 | "message": { // The user's message 167 | "role": "user", 168 | "parts": [ 169 | { 170 | "type": "text", 171 | "text": "Translate 'hello' to French" 172 | } 173 | ] 174 | } 175 | // Other optional params like sessionId could go here 176 | }, 177 | "id": "client-req-001" // Unique ID for *this specific request* 178 | } 179 | ``` 180 | 181 | If the agent accepts the task, it sends back a success response containing the initial `Task` object: 182 | 183 | ```json 184 | // Agent Sends This Back (HTTP Response body) 185 | { 186 | "jsonrpc": "2.0", 187 | "result": { // The result: a Task object 188 | "id": "task-xyz-789", // The same Task ID 189 | "status": { // The initial status 190 | "state": "submitted", 191 | "timestamp": "2023-10-27T10:00:00Z" 192 | }, 193 | "artifacts": null, // No results yet 194 | "history": null // History might be omitted initially 195 | // Other Task fields 196 | }, 197 | "id": "client-req-001" // Matches the request ID 198 | } 199 | ``` 200 | 201 | This exchange uses the JSON-RPC grammar (`method`, `params`, `result`, `id`) and the A2A vocabulary (`Task`, `Message`, `Part`, `TaskStatus`, `TaskState`) to communicate clearly. 202 | 203 | ## Handling Mistakes: Errors in the Protocol 204 | 205 | What if the client sends a request for a method the agent doesn't understand, like `tasks/make_coffee`? The agent would respond with a JSON-RPC error: 206 | 207 | ```json 208 | { 209 | "jsonrpc": "2.0", 210 | "error": { 211 | "code": -32601, // Standard JSON-RPC code for "Method not found" 212 | "message": "Method not found: tasks/make_coffee" 213 | }, 214 | "id": "client-req-002" 215 | } 216 | ``` 217 | 218 | The A2A protocol also defines some specific error codes for common agent issues: 219 | 220 | * `-32001`: `Task Not Found` (e.g., client asks for status of a task ID that doesn't exist) 221 | * `-32002`: `Task Not Cancelable` (e.g., trying to cancel an already completed task) 222 | * `-32004`: `Unsupported Operation` 223 | 224 | These standard errors help clients understand what went wrong in a predictable way. You can find definitions in the schema files: 225 | 226 | * `samples/js/src/schema.ts` (search for `ErrorCode`) 227 | * `samples/python/common/types.py` (search for error classes like `MethodNotFoundError`, `TaskNotFoundError`) 228 | 229 | ## Conclusion 230 | 231 | The A2A Protocol acts as the universal translator for AI agents. By defining: 232 | 233 | 1. A common **grammar** (JSON-RPC 2.0) for structuring requests and responses. 234 | 2. A standard **vocabulary** (Core Types like `Task`, `Message`, `Part`, `Artifact`) for the data being exchanged. 235 | 236 | ...it allows agents built by anyone, using any framework, to communicate and collaborate effectively. It ensures that when one agent asks another to do something, the request is understood, progress can be tracked, and results can be returned in a predictable format. 237 | 238 | Now that we understand the language agents speak, let's see how to build an agent that can actually listen and respond using this protocol. 239 | 240 | **Next:** [Chapter 4: A2A Server Implementation](04_a2a_server_implementation.md) 241 | 242 | --- 243 | 244 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/Google A2A/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Google A2A" 4 | nav_order: 12 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: Google A2A 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | The **Google A2A (Agent-to-Agent)**[View Repo](https://github.com/google/A2A) project defines an *open protocol* enabling different AI agents, possibly built with different technologies, to communicate and work together. 13 | Think of it as a common language (*A2A Protocol*) agents use to discover each other (*Agent Card*), assign work (*Task*), and exchange results, even providing real-time updates (*Streaming*). 14 | The project includes sample *client* and *server* implementations, example agents using frameworks like LangGraph or CrewAI, and a *demo UI* showcasing multi-agent interactions. 15 | 16 | ```mermaid 17 | flowchart TD 18 | A0["A2A Protocol & Core Types"] 19 | A1["Task"] 20 | A2["Agent Card"] 21 | A3["A2A Server Implementation"] 22 | A4["A2A Client Implementation"] 23 | A5["Task Handling Logic (Server-side)"] 24 | A6["Streaming Communication (SSE)"] 25 | A7["Demo UI Application & Service"] 26 | A8["Multi-Agent Orchestration (Host Agent)"] 27 | A0 -- "Defines Structure For" --> A1 28 | A0 -- "Defines Structure For" --> A2 29 | A4 -- "Sends Task Requests To" --> A3 30 | A3 -- "Delegates Task To" --> A5 31 | A5 -- "Executes" --> A1 32 | A8 -- "Uses for Discovery" --> A2 33 | A3 -- "Sends Updates Via" --> A6 34 | A4 -- "Receives Updates Via" --> A6 35 | A8 -- "Acts As" --> A4 36 | A7 -- "Presents/Manages" --> A8 37 | A7 -- "Communicates With" --> A5 38 | ``` -------------------------------------------------------------------------------- /docs/LangGraph/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "LangGraph" 4 | nav_order: 13 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: LangGraph 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | LangGraph[View Repo](https://github.com/langchain-ai/langgraph/tree/55f922cf2f3e63600ed8f0d0cd1262a75a991fdc/libs/langgraph/langgraph) helps you build complex **stateful applications**, like chatbots or agents, using a *graph-based approach*. 13 | You define your application's logic as a series of steps (**Nodes**) connected by transitions (**Edges**) in a **Graph**. 14 | The system manages the application's *shared state* using **Channels** and executes the graph step-by-step with its **Pregel engine**, handling things like branching, interruptions, and saving progress (**Checkpointing**). 15 | 16 | ```mermaid 17 | flowchart TD 18 | A0["Pregel Execution Engine"] 19 | A1["Graph / StateGraph"] 20 | A2["Channels"] 21 | A3["Nodes (PregelNode)"] 22 | A4["Checkpointer (BaseCheckpointSaver)"] 23 | A5["Control Flow Primitives (Branch, Send, Interrupt)"] 24 | A0 -- "Executes" --> A1 25 | A1 -- "Contains" --> A3 26 | A3 -- "Updates State Via" --> A2 27 | A0 -- "Manages State Via" --> A2 28 | A0 -- "Uses Checkpointer" --> A4 29 | A1 -- "Defines Control Flow With" --> A5 30 | A5 -- "Directs Execution Of" --> A0 31 | A4 -- "Saves State Of" --> A2 32 | ``` -------------------------------------------------------------------------------- /docs/LevelDB/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "LevelDB" 4 | nav_order: 14 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: LevelDB 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | LevelDB[View Repo](https://github.com/google/leveldb/tree/main/db) is a fast *key-value storage library* written at Google. 13 | Think of it like a simple database where you store pieces of data (values) associated with unique names (keys). 14 | It's designed to be **very fast** for both writing new data and reading existing data, and it reliably stores everything on **disk**. 15 | It uses a *log-structured merge-tree (LSM-tree)* design to achieve high write performance and manages data in sorted files (*SSTables*) across different levels for efficient reads and space management. 16 | 17 | ```mermaid 18 | flowchart TD 19 | A0["DBImpl"] 20 | A1["MemTable"] 21 | A2["Table / SSTable & TableCache"] 22 | A3["Version & VersionSet"] 23 | A4["Write-Ahead Log (WAL) & LogWriter/LogReader"] 24 | A5["Iterator"] 25 | A6["WriteBatch"] 26 | A7["Compaction"] 27 | A8["InternalKey & DBFormat"] 28 | A0 -- "Manages active/immutable" --> A1 29 | A0 -- "Uses Cache for reads" --> A2 30 | A0 -- "Manages DB state" --> A3 31 | A0 -- "Writes to Log" --> A4 32 | A0 -- "Applies Batches" --> A6 33 | A0 -- "Triggers/Runs Compaction" --> A7 34 | A1 -- "Provides Iterator" --> A5 35 | A1 -- "Stores Keys Using" --> A8 36 | A2 -- "Provides Iterator via Cache" --> A5 37 | A3 -- "References SSTables" --> A2 38 | A3 -- "Picks Files For" --> A7 39 | A4 -- "Recovers MemTable From" --> A1 40 | A4 -- "Contains Batch Data" --> A6 41 | A5 -- "Parses/Hides InternalKey" --> A8 42 | A6 -- "Inserts Into" --> A1 43 | A7 -- "Builds SSTables" --> A2 44 | A7 -- "Updates Versions Via Edit" --> A3 45 | A7 -- "Uses Iterator for Merging" --> A5 46 | ``` -------------------------------------------------------------------------------- /docs/MCP Python SDK/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "MCP Python SDK" 4 | nav_order: 15 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: MCP Python SDK 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | The **MCP Python SDK**[View Repo](https://github.com/modelcontextprotocol/python-sdk/tree/d788424caa43599de38cee2f70233282d83e3a34/src/mcp) helps developers build applications (clients and servers) that talk to each other using the *Model Context Protocol (MCP)* specification. 13 | It simplifies communication by handling the low-level details like standard **message formats** (Abstraction 0), connection **sessions** (Abstraction 1), and different ways to send/receive data (**transports**, Abstraction 2). 14 | It also provides a high-level framework, **`FastMCP`** (Abstraction 3), making it easy to create servers that expose **tools** (Abstraction 5), **resources** (Abstraction 4), and **prompts** (Abstraction 6) to clients. 15 | The SDK includes **command-line tools** (Abstraction 8) for running and managing these servers. 16 | 17 | ```mermaid 18 | flowchart TD 19 | A0["MCP Protocol Types"] 20 | A1["Client/Server Sessions"] 21 | A2["Communication Transports"] 22 | A3["FastMCP Server"] 23 | A4["FastMCP Resources"] 24 | A5["FastMCP Tools"] 25 | A6["FastMCP Prompts"] 26 | A7["FastMCP Context"] 27 | A8["CLI"] 28 | A1 -- "Uses MCP Types" --> A0 29 | A1 -- "Operates Over Transport" --> A2 30 | A2 -- "Serializes/Deserializes MCP..." --> A0 31 | A3 -- "Uses Session Logic" --> A1 32 | A3 -- "Manages Resources" --> A4 33 | A3 -- "Manages Tools" --> A5 34 | A3 -- "Manages Prompts" --> A6 35 | A8 -- "Runs/Configures Server" --> A3 36 | A5 -- "Handlers Can Use Context" --> A7 37 | A4 -- "Handlers Can Use Context" --> A7 38 | A7 -- "Provides Access To Session" --> A1 39 | A7 -- "Provides Access To Server" --> A3 40 | ``` 41 | -------------------------------------------------------------------------------- /docs/NumPy Core/01_ndarray__n_dimensional_array_.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "ndarray (N-dimensional array)" 4 | parent: "NumPy Core" 5 | nav_order: 1 6 | --- 7 | 8 | # Chapter 1: ndarray (N-dimensional array) 9 | 10 | Welcome to the NumPy Core tutorial! If you're interested in how NumPy works under the hood, you're in the right place. NumPy is the foundation for scientific computing in Python, and its core strength comes from a special object called the `ndarray`. 11 | 12 | Imagine you have a huge list of numbers, maybe temperatures recorded every second for a year, or the pixel values of a large image. Doing math with standard Python lists can be quite slow for these large datasets. This is the problem NumPy, and specifically the `ndarray`, is designed to solve. 13 | 14 | ## What is an ndarray? 15 | 16 | Think of an `ndarray` (which stands for N-dimensional array) as a powerful grid or table designed to hold items **of the same type**, usually numbers (like integers or decimals). It's the fundamental building block of NumPy. 17 | 18 | * **Grid:** It can be a simple list (1-dimension), a table with rows and columns (2-dimensions), or even have more dimensions (3D, 4D, ... N-D). 19 | * **Same Type:** This is key! Unlike Python lists that can hold anything (numbers, strings, objects), NumPy arrays require all elements to be of the *same data type* (e.g., all 32-bit integers or all 64-bit floating-point numbers). This restriction allows NumPy to store and operate on the data extremely efficiently. We'll explore data types more in [Chapter 2: dtype (Data Type Object)](02_dtype__data_type_object_.md). 20 | 21 | Analogy: Think of a Python list as a drawer where you can throw anything in – socks, books, tools. An `ndarray` is like a specialized toolbox or an egg carton – designed to hold only specific things (only tools, only eggs) in an organized way. This organization makes it much faster to work with. 22 | 23 | Here's a quick peek at what different dimensional arrays look like conceptually: 24 | 25 | ```mermaid 26 | flowchart LR 27 | A[0] --> B[1] --> C[2] --> D[3] 28 | ``` 29 | 30 | ```mermaid 31 | flowchart LR 32 | subgraph Row 1 33 | R1C1[ R1C1 ] --> R1C2[ R1C2 ] --> R1C3[ R1C3 ] 34 | end 35 | 36 | subgraph Row 2 37 | R2C1[ R2C1 ] --> R2C2[ R2C2 ] --> R2C3[ R2C3 ] 38 | end 39 | 40 | R1C1 -.-> R2C1 41 | R1C2 -.-> R2C2 42 | R1C3 -.-> R2C3 43 | ``` 44 | 45 | ```mermaid 46 | flowchart LR 47 | subgraph Layer 1 48 | L1R1C1[ L1R1C1 ] --> L1R1C2[ L1R1C2 ] 49 | L1R2C1[ L1R2C1 ] --> L1R2C2[ L1R2C2 ] 50 | L1R1C1 -.-> L1R2C1 51 | L1R1C2 -.-> L1R2C2 52 | end 53 | 54 | subgraph Layer 2 55 | L2R1C1[ L2R1C1 ] --> L2R1C2[ L2R1C2 ] 56 | L2R2C1[ L2R2C1 ] --> L2R2C2[ L2R2C2 ] 57 | L2R1C1 -.-> L2R2C1 58 | L2R1C2 -.-> L2R2C2 59 | end 60 | 61 | L1R1C1 --- L2R1C1 62 | L1R1C2 --- L2R1C2 63 | L1R2C1 --- L2R2C1 64 | L1R2C2 --- L2R2C2 65 | ``` 66 | 67 | 68 | 69 | ## Why ndarrays? The Magic of Vectorization 70 | 71 | Let's say you have two lists of numbers and you want to add them element by element. In standard Python, you'd use a loop: 72 | 73 | ```python 74 | # Using standard Python lists 75 | list1 = [1, 2, 3, 4] 76 | list2 = [5, 6, 7, 8] 77 | result = [] 78 | for i in range(len(list1)): 79 | result.append(list1[i] + list2[i]) 80 | 81 | print(result) 82 | # Output: [6, 8, 10, 12] 83 | ``` 84 | This works, but for millions of numbers, this Python loop becomes slow. 85 | 86 | Now, see how you do it with NumPy ndarrays: 87 | 88 | ```python 89 | import numpy as np # Standard way to import NumPy 90 | 91 | array1 = np.array([1, 2, 3, 4]) 92 | array2 = np.array([5, 6, 7, 8]) 93 | 94 | # Add the arrays directly! 95 | result_array = array1 + array2 96 | 97 | print(result_array) 98 | # Output: [ 6 8 10 12] 99 | ``` 100 | Notice how we just used `+` directly on the arrays? This is called **vectorization**. You write the operation as if you're working on single values, but NumPy applies it to *all* elements automatically. 101 | 102 | **Why is this better?** 103 | 104 | 1. **Speed:** The looping happens behind the scenes in highly optimized C code, which is *much* faster than a Python loop. 105 | 2. **Readability:** The code is cleaner and looks more like standard mathematical notation. 106 | 107 | This ability to perform operations on entire arrays at once is a core reason why NumPy is so powerful and widely used. 108 | 109 | ## Creating Your First ndarrays 110 | 111 | Let's create some arrays. First, we always import NumPy, usually as `np`: 112 | 113 | ```python 114 | import numpy as np 115 | ``` 116 | 117 | **1. From Python Lists:** The most common way is using `np.array()`: 118 | 119 | ```python 120 | # Create a 1-dimensional array (vector) 121 | my_list = [10, 20, 30] 122 | arr1d = np.array(my_list) 123 | print(arr1d) 124 | # Output: [10 20 30] 125 | 126 | # Create a 2-dimensional array (matrix/table) 127 | my_nested_list = [[1, 2, 3], [4, 5, 6]] 128 | arr2d = np.array(my_nested_list) 129 | print(arr2d) 130 | # Output: 131 | # [[1 2 3] 132 | # [4 5 6]] 133 | ``` 134 | `np.array()` takes your list (or list of lists) and converts it into an ndarray. NumPy tries to figure out the best data type automatically. 135 | 136 | **2. Arrays of Zeros or Ones:** Often useful as placeholders. 137 | 138 | ```python 139 | # Create an array of shape (2, 3) filled with zeros 140 | zeros_arr = np.zeros((2, 3)) 141 | print(zeros_arr) 142 | # Output: 143 | # [[0. 0. 0.] 144 | # [0. 0. 0.]] 145 | 146 | # Create an array of shape (3,) filled with ones 147 | ones_arr = np.ones(3) 148 | print(ones_arr) 149 | # Output: [1. 1. 1.] 150 | ``` 151 | Notice we pass a tuple like `(2, 3)` to specify the desired shape. By default, these are filled with floating-point numbers. 152 | 153 | **3. Using `np.arange`:** Similar to Python's `range`. 154 | 155 | ```python 156 | # Create an array with numbers from 0 up to (but not including) 5 157 | range_arr = np.arange(5) 158 | print(range_arr) 159 | # Output: [0 1 2 3 4] 160 | ``` 161 | 162 | There are many other ways to create arrays, but these are fundamental. 163 | 164 | ## Exploring Your ndarray: Basic Attributes 165 | 166 | Once you have an array, you can easily check its properties: 167 | 168 | ```python 169 | arr = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) 170 | 171 | # 1. Shape: The size of each dimension 172 | print(f"Shape: {arr.shape}") 173 | # Output: Shape: (2, 3) (2 rows, 3 columns) 174 | 175 | # 2. Number of Dimensions (ndim): How many axes it has 176 | print(f"Dimensions: {arr.ndim}") 177 | # Output: Dimensions: 2 178 | 179 | # 3. Size: Total number of elements 180 | print(f"Size: {arr.size}") 181 | # Output: Size: 6 182 | 183 | # 4. Data Type (dtype): The type of elements in the array 184 | print(f"Data Type: {arr.dtype}") 185 | # Output: Data Type: float64 186 | ``` 187 | These attributes are crucial for understanding the structure of your data. The `dtype` tells you what kind of data is stored (e.g., `int32`, `float64`, `bool`). We'll dive much deeper into this in [Chapter 2: dtype (Data Type Object)](02_dtype__data_type_object_.md). 188 | 189 | ## A Glimpse Under the Hood 190 | 191 | So, how does NumPy achieve its speed? The `ndarray` you manipulate in Python is actually a clever wrapper around a highly efficient data structure implemented in the **C programming language**. 192 | 193 | When you perform an operation like `array1 + array2`, Python doesn't slowly loop through the elements. Instead, NumPy: 194 | 195 | 1. Checks if the operation is valid (e.g., arrays are compatible). 196 | 2. Hands off the arrays and the operation (`+` in this case) to its underlying C code. 197 | 3. The C code, which is pre-compiled and highly optimized for your processor, performs the addition very rapidly across the entire block of memory holding the array data. 198 | 4. The result (another block of memory) is then wrapped back into a new Python `ndarray` object for you to use. 199 | 200 | Here's a simplified view of what happens when you call `np.array()`: 201 | 202 | ```mermaid 203 | sequenceDiagram 204 | participant P as Python Code (Your script) 205 | participant NPF as NumPy Python Function (e.g., np.array) 206 | participant CF as C Function (in _multiarray_umath) 207 | participant M as Memory 208 | 209 | P->>NPF: np.array([1, 2, 3]) 210 | NPF->>CF: Call C implementation with list data 211 | CF->>M: Allocate contiguous memory block 212 | CF->>M: Copy data [1, 2, 3] into block 213 | CF-->>NPF: Return C-level ndarray structure pointing to memory 214 | NPF-->>P: Return Python ndarray object wrapping the C structure 215 | ``` 216 | 217 | The core implementation lives within compiled C extension modules, primarily `_multiarray_umath`. Python files like `numpy/core/multiarray.py` and `numpy/core/numeric.py` provide the convenient Python functions (`np.array`, `np.zeros`, etc.) that eventually call this fast C code. You can see how `numeric.py` imports functions from `multiarray`: 218 | 219 | ```python 220 | # From numpy/core/numeric.py - Simplified 221 | from . import multiarray 222 | from .multiarray import ( 223 | arange, array, asarray, asanyarray, # <-- Python functions defined here 224 | empty, empty_like, zeros # <-- More functions 225 | # ... many others ... 226 | ) 227 | 228 | # The `array` function seen in multiarray.py is often a wrapper 229 | # that calls the actual C implementation. 230 | ``` 231 | This setup gives you the ease of Python with the speed of C. The `ndarray` object itself stores metadata (like shape, dtype, strides) and a pointer to the actual raw data block in memory. We will see more details about the Python modules involved in [Chapter 6: multiarray Module](06_multiarray_module.md) and [Chapter 7: umath Module](07_umath_module.md). 232 | 233 | ## Conclusion 234 | 235 | You've met the `ndarray`, the heart of NumPy! You learned: 236 | 237 | * It's a powerful, efficient grid for storing elements of the **same type**. 238 | * It enables **vectorization**, allowing fast operations on entire arrays without explicit Python loops. 239 | * How to create basic arrays using `np.array`, `np.zeros`, `np.ones`, and `np.arange`. 240 | * How to check key properties like `shape`, `ndim`, `size`, and `dtype`. 241 | * That the speed comes from an underlying **C implementation**. 242 | 243 | The `ndarray` is the container. Now, let's look more closely at *what* it contains – the different types of data it can hold. 244 | 245 | Ready to learn about data types? Let's move on to [Chapter 2: dtype (Data Type Object)](02_dtype__data_type_object_.md). 246 | 247 | --- 248 | 249 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 250 | -------------------------------------------------------------------------------- /docs/NumPy Core/06_multiarray_module.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Multiarray Module" 4 | parent: "NumPy Core" 5 | nav_order: 6 6 | --- 7 | 8 | # Chapter 6: multiarray Module 9 | 10 | Welcome back! In [Chapter 5: Array Printing (`arrayprint`)](05_array_printing___arrayprint__.md), we saw how NumPy takes complex arrays and presents them in a readable format. We've now covered the array container ([`ndarray`](01_ndarray__n_dimensional_array_.md)), its data types ([`dtype`](02_dtype__data_type_object_.md)), the functions that compute on them ([`ufunc`](03_ufunc__universal_function_.md)), the catalog of types ([`numerictypes`](04_numeric_types___numerictypes__.md)), and how arrays are displayed ([`arrayprint`](05_array_printing___arrayprint__.md)). 11 | 12 | Now, let's peek deeper into the engine room. Where does the fundamental `ndarray` object *actually* come from? How are core operations like creating arrays or accessing elements implemented so efficiently? The answer lies largely within the C code associated with the concept of the `multiarray` module. 13 | 14 | ## What Problem Does `multiarray` Solve? Providing the Engine 15 | 16 | Think about the very first step in using NumPy: creating an array. 17 | 18 | ```python 19 | import numpy as np 20 | 21 | # How does this seemingly simple line actually work? 22 | my_array = np.array([1, 2, 3, 4, 5]) 23 | 24 | # How does NumPy know its shape? How is the data stored? 25 | print(my_array) 26 | print(my_array.shape) 27 | ``` 28 | 29 | When you execute `np.array()`, you're using a convenient Python function. But NumPy's speed doesn't come from Python itself. It comes from highly optimized code written in the C programming language. How do these Python functions connect to that fast C code? And where is that C code defined? 30 | 31 | The `multiarray` concept represents this core C engine. It's the part of NumPy responsible for: 32 | 33 | 1. **Defining the `ndarray` object:** The very structure that holds your data, its shape, its data type ([`dtype`](02_dtype__data_type_object_.md)), and how it's laid out in memory. 34 | 2. **Implementing Fundamental Operations:** Providing the low-level C functions for creating arrays (like allocating memory), accessing elements (indexing), changing the view (slicing, reshaping), and basic mathematical operations. 35 | 36 | Think of the Python functions like `np.array`, `np.zeros`, or accessing `arr.shape` as the dashboard and controls of a car. The `multiarray` C code is the powerful engine under the hood that actually makes the car move efficiently. 37 | 38 | ## What is the `multiarray` Module (Concept)? 39 | 40 | Historically, `multiarray` was a distinct C extension module in NumPy. An "extension module" is a module written in C (or C++) that Python can import and use just like a regular Python module. This allows Python code to leverage the speed of C for performance-critical tasks. 41 | 42 | More recently (since NumPy 1.16), the C code for `multiarray` was merged with the C code for the [ufunc (Universal Function)](03_ufunc__universal_function_.md) system (which we'll discuss more in [Chapter 7: umath Module](07_umath_module.md)) into a single, larger C extension module typically called `_multiarray_umath.cpython-*.so` (on Linux/Mac) or `_multiarray_umath.pyd` (on Windows). 43 | 44 | Even though the C code is merged, the *concept* of `multiarray` remains important. It represents the C implementation layer that provides: 45 | 46 | * The **`ndarray` object type** itself (`PyArrayObject` in C). 47 | * The **C-API (Application Programming Interface)**: A set of C functions that can be called by other C extensions (and internally by NumPy's Python code) to work with `ndarray` objects. Examples include functions to create arrays from data, get the shape, get the data pointer, perform indexing, etc. 48 | * Implementations of **core array functionalities**: array creation, data type handling ([`dtype`](02_dtype__data_type_object_.md)), memory layout management (strides), indexing, slicing, reshaping, transposing, and some basic operations. 49 | 50 | The Python files you might see in the NumPy source code, like `numpy/core/multiarray.py` and `numpy/core/numeric.py`, often serve as Python wrappers. They provide the user-friendly Python functions (like `np.array`, `np.empty`, `np.dot`) that eventually call the fast C functions implemented within the `_multiarray_umath` extension module. 51 | 52 | ```python 53 | # numpy/core/multiarray.py - Simplified Example 54 | # This Python file imports directly from the C extension module 55 | 56 | from . import _multiarray_umath # Import the compiled C module 57 | from ._multiarray_umath import * # Make C functions available 58 | 59 | # Functions like 'array', 'empty', 'dot' that you use via `np.` 60 | # might be defined or re-exported here, ultimately calling C code. 61 | # For example, the `array` function here might parse the Python input 62 | # and then call a C function like `PyArray_NewFromDescr` from _multiarray_umath. 63 | ``` 64 | 65 | This structure gives you the flexibility and ease of Python on the surface, powered by the speed and efficiency of C underneath. 66 | 67 | ## A Glimpse Under the Hood: Creating an Array 68 | 69 | Let's trace what happens when you call `my_array = np.array([1, 2, 3])`: 70 | 71 | 1. **Python Call:** You call the Python function `np.array`. This function likely lives in `numpy/core/numeric.py` or is exposed through `numpy/core/multiarray.py`. 72 | 2. **Argument Parsing:** The Python function examines the input `[1, 2, 3]`. It figures out the data type (likely `int64` by default on many systems) and the shape (which is `(3,)`). 73 | 3. **Call C-API Function:** The Python function calls a specific function within the compiled `_multiarray_umath` C extension module. This C function is designed to create a new array. A common one is `PyArray_NewFromDescr` or a related helper. 74 | 4. **Memory Allocation (C):** The C function asks the operating system for a block of memory large enough to hold 3 integers of the chosen type (e.g., 3 * 8 bytes = 24 bytes for `int64`). 75 | 5. **Data Copying (C):** The C function copies the values `1`, `2`, and `3` from the Python list into the newly allocated memory block. 76 | 6. **Create C `ndarray` Struct:** The C function creates an internal C structure (called `PyArrayObject`). This structure stores: 77 | * A pointer to the actual data block in memory. 78 | * Information about the data type ([`dtype`](02_dtype__data_type_object_.md)). 79 | * The shape of the array (`(3,)`). 80 | * The strides (how many bytes to jump to get to the next element in each dimension). 81 | * Other metadata (like flags indicating if it owns the data, if it's writeable, etc.). 82 | 7. **Wrap in Python Object:** The C function wraps this internal `PyArrayObject` structure into a Python object that Python can understand – the `ndarray` object you interact with. 83 | 8. **Return to Python:** The C function returns this new Python `ndarray` object back to your Python code, which assigns it to the variable `my_array`. 84 | 85 | Here's a simplified view of that flow: 86 | 87 | ```mermaid 88 | sequenceDiagram 89 | participant User as Your Python Script 90 | participant PyFunc as NumPy Python Func (np.array) 91 | participant C_API as C Code (_multiarray_umath) 92 | participant Memory 93 | 94 | User->>PyFunc: my_array = np.array([1, 2, 3]) 95 | PyFunc->>C_API: Call C function (e.g., PyArray_NewFromDescr) with list data, inferred dtype, shape 96 | C_API->>Memory: Allocate memory block (e.g., 24 bytes for 3x int64) 97 | C_API->>Memory: Copy data [1, 2, 3] into block 98 | C_API->>C_API: Create internal C ndarray struct (PyArrayObject) pointing to data, storing shape=(3,), dtype=int64, etc. 99 | C_API->>PyFunc: Return Python ndarray object wrapping the C struct 100 | PyFunc-->>User: Assign returned ndarray object to `my_array` 101 | ``` 102 | 103 | **Where is the Code?** 104 | 105 | * **C Implementation:** The core logic is in C files compiled into the `_multiarray_umath` extension module (e.g., parts of `numpy/core/src/multiarray/`). Files like `alloc.c`, `ctors.c` (constructors), `getset.c` (for getting/setting attributes like shape), `item_selection.c` (indexing) contain relevant C code. 106 | * **Python Wrappers:** `numpy/core/numeric.py` and `numpy/core/multiarray.py` provide many of the familiar Python functions. They import directly from `_multiarray_umath`. 107 | ```python 108 | # From numpy/core/numeric.py - Simplified 109 | from . import multiarray # Imports numpy/core/multiarray.py 110 | # multiarray.py itself imports from _multiarray_umath 111 | from .multiarray import ( 112 | array, asarray, zeros, empty, # Functions defined/re-exported 113 | # ... many others ... 114 | ) 115 | ``` 116 | * **Initialization:** `numpy/core/__init__.py` helps set up the `numpy.core` namespace, importing from `multiarray` and `umath`. 117 | ```python 118 | # From numpy/core/__init__.py - Simplified 119 | from . import multiarray 120 | from . import umath 121 | # ... other imports ... 122 | from . import numeric 123 | from .numeric import * # Pulls in functions like np.array, np.zeros 124 | # ... more setup ... 125 | ``` 126 | * **C API Definition:** Files like `numpy/core/include/numpy/multiarray.h` define the C structures (`PyArrayObject`) and function prototypes (`PyArray_NewFromDescr`, etc.) that make up the NumPy C-API. Code generators like `numpy/core/code_generators/generate_numpy_api.py` help create tables (`__multiarray_api.h`, `__multiarray_api.c`) that allow other C extensions to easily access these core NumPy C functions. 127 | ```python 128 | # Snippet from numpy/core/code_generators/generate_numpy_api.py 129 | # This script generates C code that defines an array of function pointers 130 | # making up the C-API. 131 | 132 | # Describes API functions, their index in the API table, return type, args... 133 | multiarray_funcs = { 134 | # ... many functions ... 135 | 'NewLikeArray': (10, None, 'PyObject *', (('PyArrayObject *', 'prototype'), ...)), 136 | 'NewFromDescr': (9, None, 'PyObject *', ...), 137 | 'Empty': (8, None, 'PyObject *', ...), 138 | # ... 139 | } 140 | 141 | # ... code to generate C header (.h) and implementation (.c) files ... 142 | # These generated files help expose the C functions consistently. 143 | ``` 144 | 145 | ## Conclusion 146 | 147 | You've now learned about the conceptual `multiarray` module, the C engine at the heart of NumPy. 148 | 149 | * It's implemented in **C** (as part of the `_multiarray_umath` extension module) for maximum **speed and efficiency**. 150 | * It provides the fundamental **`ndarray` object** structure. 151 | * It implements **core array operations** like creation, memory management, indexing, and reshaping at a low level. 152 | * Python modules like `numpy.core.numeric` and `numpy.core.multiarray` provide user-friendly interfaces that call this underlying C code. 153 | * Understanding this separation helps explain *why* NumPy is so fast compared to standard Python lists for numerical tasks. 154 | 155 | While `multiarray` provides the array structure and basic manipulation, the element-wise mathematical operations often rely on another closely related C implementation layer. 156 | 157 | Let's explore that next in [Chapter 7: umath Module](07_umath_module.md). 158 | 159 | --- 160 | 161 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/NumPy Core/07_umath_module.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Umath Module" 4 | parent: "NumPy Core" 5 | nav_order: 7 6 | --- 7 | 8 | # Chapter 7: umath Module 9 | 10 | Welcome to Chapter 7! In [Chapter 6: multiarray Module](06_multiarray_module.md), we explored the core C engine that defines the `ndarray` object and handles fundamental operations like creating arrays and accessing elements. We saw that the actual power comes from C code. 11 | 12 | But what about the mathematical operations themselves? When you perform `np.sin(my_array)` or `array1 + array2`, which part of the C engine handles the actual sine calculation or the addition for *every single element*? This is where the concept of the `umath` module comes in. 13 | 14 | ## What Problem Does `umath` Solve? Implementing Fast Array Math 15 | 16 | Remember the [ufunc (Universal Function)](03_ufunc__universal_function_.md) from Chapter 3? Ufuncs are NumPy's special functions designed to operate element-wise on arrays with incredible speed (like `np.add`, `np.sin`, `np.log`). 17 | 18 | Let's take a simple example: 19 | 20 | ```python 21 | import numpy as np 22 | 23 | angles = np.array([0, np.pi/2, np.pi]) 24 | sines = np.sin(angles) # How is this sine calculated so fast? 25 | 26 | print(angles) 27 | print(sines) 28 | ``` 29 | 30 | **Output:** 31 | 32 | ``` 33 | [0. 1.57079633 3.14159265] 34 | [0.0000000e+00 1.0000000e+00 1.2246468e-16] # Note: pi value is approximate 35 | ``` 36 | 37 | The Python function `np.sin` acts as a dispatcher. It needs to hand off the actual, heavy-duty work of calculating the sine for each element in the `angles` array to highly optimized code. Where does this optimized code live? 38 | 39 | Historically, the C code responsible for implementing the *loops and logic* of these mathematical ufuncs (like addition, subtraction, sine, cosine, logarithm, etc.) was contained within a dedicated C extension module called `umath`. It provided the fast, element-by-element computational kernels. 40 | 41 | ## What is the `umath` Module (Concept)? 42 | 43 | The `umath` module represents the part of NumPy's C core dedicated to implementing **universal functions (ufuncs)**. Think of it as NumPy's built-in, highly optimized math library specifically designed for element-wise operations on arrays. 44 | 45 | **Key Points:** 46 | 47 | 1. **Houses ufunc Implementations:** It contains the low-level C code that performs the actual calculations for functions like `np.add`, `np.sin`, `np.exp`, `np.sqrt`, etc. 48 | 2. **Optimized Loops:** This C code includes specialized loops that iterate over the array elements very efficiently, often tailored for specific [dtype (Data Type Object)](02_dtype__data_type_object_.md)s (like a fast loop for adding 32-bit integers, another for 64-bit floats, etc.). 49 | 3. **Historical C Module:** Originally, `umath` was a separate compiled C extension module (`umath.so` or `umath.pyd`). 50 | 4. **Merged with `multiarray`:** Since NumPy 1.16, the C code for `umath` has been merged with the C code for `multiarray` into a single, larger C extension module named `_multiarray_umath`. While they are now in the same compiled file, the *functions and purpose* associated with `umath` (implementing ufunc math) are distinct from those associated with `multiarray` (array object structure and basic manipulation). 51 | 5. **Python Access (`numpy/core/umath.py`):** You don't usually interact with the C code directly. Instead, NumPy provides Python functions (like `np.add`, `np.sin`) in the Python file `numpy/core/umath.py`. These Python functions are wrappers that know how to find and trigger the correct C implementation within the `_multiarray_umath` extension module. 52 | 53 | **Analogy:** Imagine `multiarray` builds the car chassis and engine block (`ndarray` structure). `umath` provides specialized, high-performance engine components like the fuel injectors for addition (`np.add`'s C code), the turbocharger for exponentiation (`np.exp`'s C code), and the precise valve timing for trigonometry (`np.sin`'s C code). The Python functions (`np.add`, `np.sin`) are the pedals and buttons you use to activate these components. 54 | 55 | ## How it Works (Usage Perspective) 56 | 57 | As a NumPy user, you typically trigger the `umath` C code indirectly by calling a ufunc: 58 | 59 | ```python 60 | import numpy as np 61 | 62 | a = np.array([1, 2, 3]) 63 | b = np.array([10, 20, 30]) 64 | 65 | # Calling the ufunc np.add 66 | result1 = np.add(a, b) # Triggers the C implementation for addition 67 | 68 | # Using the operator '+' which also calls np.add for arrays 69 | result2 = a + b # Also triggers the C implementation 70 | 71 | print(f"Using np.add: {result1}") 72 | print(f"Using + operator: {result2}") 73 | ``` 74 | 75 | **Output:** 76 | 77 | ``` 78 | Using np.add: [11 22 33] 79 | Using + operator: [11 22 33] 80 | ``` 81 | 82 | Both `np.add(a, b)` and `a + b` ultimately lead to NumPy executing the highly optimized C code associated with the addition ufunc, which conceptually belongs to the `umath` part of the core. 83 | 84 | ## A Glimpse Under the Hood 85 | 86 | When you call a ufunc like `np.add(a, b)`: 87 | 88 | 1. **Python Call:** You invoke the Python function `np.add` (found in `numpy/core/umath.py` or exposed through `numpy/core/__init__.py`). 89 | 2. **Identify Ufunc Object:** This Python function accesses the corresponding ufunc object (`np.add` itself is a ufunc object). This object holds metadata about the operation. 90 | 3. **Dispatch to C:** The ufunc object mechanism (part of the `_multiarray_umath` C core) takes over. 91 | 4. **Type Resolution & Loop Selection:** The C code inspects the `dtype`s of the input arrays (`a` and `b`). Based on the input types, it looks up an internal table associated with the `add` ufunc to find the *best* matching, pre-compiled C loop. For example, if `a` and `b` are both `int64`, it selects the C function specifically designed for `int64 + int64 -> int64`. This selection process might involve type casting rules (e.g., adding `int32` and `float64` might choose a loop that operates on `float64`). 92 | 5. **Execute C Loop:** The selected C function (the core `umath` implementation for this specific type combination) is executed. This function iterates efficiently over the input array(s) memory, performs the addition element by element, and stores the results in the output array's memory. 93 | 6. **Return Result:** The C machinery wraps the output memory into a new `ndarray` object and returns it back to your Python code. 94 | 95 | Here's a simplified sequence diagram: 96 | 97 | ```mermaid 98 | sequenceDiagram 99 | participant User as Your Python Script 100 | participant PyUfunc as np.add (Python Wrapper) 101 | participant UfuncObj as Ufunc Object (Metadata) 102 | participant C_Core as C Code (_multiarray_umath) 103 | participant C_Loop as Specific Add Loop (e.g., int64_add) 104 | participant Memory 105 | 106 | User->>PyUfunc: result = np.add(a, b) 107 | PyUfunc->>UfuncObj: Access the 'add' ufunc object 108 | UfuncObj->>C_Core: Initiate ufunc execution (pass inputs a, b) 109 | C_Core->>C_Core: Inspect a.dtype, b.dtype 110 | C_Core->>UfuncObj: Find best C loop (e.g., int64_add loop) 111 | C_Core->>Memory: Allocate memory for result (if needed) 112 | C_Core->>C_Loop: Execute int64_add(a_data, b_data, result_data) 113 | C_Loop->>Memory: Read a, b, compute sum, write result 114 | C_Loop-->>C_Core: Signal loop completion 115 | C_Core->>Memory: Wrap result memory in ndarray object 116 | C_Core-->>PyUfunc: Return result ndarray 117 | PyUfunc-->>User: Assign result ndarray to 'result' 118 | 119 | ``` 120 | 121 | **Where is the Code?** 122 | 123 | * **C Extension Module:** The compiled code lives in `_multiarray_umath.so` / `.pyd`. 124 | * **Ufunc Definition & Generation:** The script `numpy/core/code_generators/generate_umath.py` is crucial. It contains definitions (like the `defdict` dictionary) that describe each ufunc: its name, number of inputs/outputs, identity element, the C functions to use for different type combinations (`TD` entries), and associated docstrings. This script generates C code (`__umath_generated.c`, which is then compiled) that sets up the ufunc objects and their internal loop tables. 125 | ```python 126 | # Simplified snippet from generate_umath.py's defdict for 'add' 127 | 'add': 128 | Ufunc(2, 1, Zero, # nin=2, nout=1, identity=0 129 | docstrings.get('numpy._core.umath.add'), # Docstring reference 130 | 'PyUFunc_AdditionTypeResolver', # Type resolution logic 131 | TD('?', ...), # Loop for booleans 132 | TD(no_bool_times_obj, dispatch=[...]), # Loops for numeric types 133 | # ... loops for datetime, object ... 134 | ), 135 | ``` 136 | This definition tells the generator how to build the `np.add` ufunc, including which C functions (often defined in other C files or generated from templates) handle addition for different data types. 137 | * **C Loop Implementations:** The actual C code performing the math often comes from template files (like `numpy/core/src/umath/loops.c.src`) or CPU-dispatch-specific files (like `numpy/core/src/umath/loops_arithm_fp.dispatch.c.src`). These `.src` files contain templates written in a C-like syntax that get processed to generate specific C code for various data types (e.g., generating `int32_add`, `int64_add`, `float32_add`, `float64_add` from a single addition template). The dispatch files allow NumPy to choose optimized code paths (using e.g., AVX2, AVX512 instructions) based on your CPU's capabilities at runtime. 138 | * **Python Wrappers:** `numpy/core/umath.py` provides the Python functions like `np.add`, `np.sin` that you call. It primarily imports these functions directly from the `_multiarray_umath` C extension module. 139 | ```python 140 | # From numpy/core/umath.py - Simplified 141 | from . import _multiarray_umath 142 | from ._multiarray_umath import * # Imports C-defined ufuncs like 'add' 143 | 144 | # Functions like 'add', 'sin', 'log' are now available in this module's 145 | # namespace, ready to be used via `np.add`, `np.sin`, etc. 146 | ``` 147 | * **Namespace Setup:** `numpy/core/__init__.py` imports from `numpy.core.umath` (among others) to make functions like `np.add` easily accessible under the main `np` namespace. 148 | 149 | ## Conclusion 150 | 151 | You've now seen that the `umath` concept represents the implementation heart of NumPy's universal functions. 152 | 153 | * It provides the optimized **C code** that performs element-wise mathematical operations. 154 | * It contains specialized **loops** for different data types, crucial for NumPy's speed. 155 | * While historically a separate C module, its functionality is now part of the merged `_multiarray_umath` C extension. 156 | * Python files like `numpy/core/umath.py` provide access, but the real work happens in C, often defined via generators like `generate_umath.py` and implemented in templated `.src` or dispatchable C files. 157 | 158 | Understanding `umath` clarifies where the computational power for element-wise operations originates within NumPy's core. 159 | 160 | So far, we've focused on NumPy's built-in functions. But how does NumPy interact with other libraries or allow customization of how operations work on its arrays? 161 | 162 | Next, we'll explore a powerful mechanism for extending NumPy's reach: [Chapter 8: __array_function__ Protocol / Overrides (`overrides`)](08___array_function___protocol___overrides___overrides__.md). 163 | 164 | --- 165 | 166 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 167 | -------------------------------------------------------------------------------- /docs/NumPy Core/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "NumPy Core" 4 | nav_order: 16 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: NumPy Core 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | NumPy Core[View Repo](https://github.com/numpy/numpy/tree/3b377854e8b1a55f15bda6f1166fe9954828231b/numpy/_core) provides the powerful **ndarray** object, a *multi-dimensional grid* optimized for numerical computations on large datasets. It uses **dtypes** (data type objects) to precisely define the *kind of data* (like integers or floating-point numbers) stored within an array, ensuring memory efficiency and enabling optimized low-level operations. NumPy also features **ufuncs** (universal functions), which are functions like `add` or `sin` designed to operate *element-wise* on entire arrays very quickly, leveraging compiled code. Together, these components form the foundation for high-performance scientific computing in Python. 13 | 14 | ```mermaid 15 | flowchart TD 16 | A0["ndarray (N-dimensional array)"] 17 | A1["dtype (Data Type Object)"] 18 | A2["ufunc (Universal Function)"] 19 | A3["multiarray Module"] 20 | A4["umath Module"] 21 | A5["Numeric Types"] 22 | A6["Array Printing"] 23 | A7["__array_function__ Protocol / Overrides"] 24 | A0 -- "Has data type" --> A1 25 | A2 -- "Operates element-wise on" --> A0 26 | A3 -- "Provides implementation for" --> A0 27 | A4 -- "Provides implementation for" --> A2 28 | A5 -- "Defines scalar types for" --> A1 29 | A6 -- "Formats for display" --> A0 30 | A6 -- "Uses for formatting info" --> A1 31 | A7 -- "Overrides functions from" --> A3 32 | A7 -- "Overrides functions from" --> A4 33 | A1 -- "References type hierarchy" --> A5 34 | ``` -------------------------------------------------------------------------------- /docs/OpenManus/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "OpenManus" 4 | nav_order: 17 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: OpenManus 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | OpenManus[View Repo](https://github.com/mannaandpoem/OpenManus/tree/f616c5d43d02d93ccc6e55f11666726d6645fdc2) is a framework for building autonomous *AI agents*. 13 | Think of it like a digital assistant that can perform tasks. It uses a central **brain** (an `LLM` like GPT-4) to understand requests and decide what to do next. 14 | Agents can use various **tools** (like searching the web or writing code) to interact with the world or perform specific actions. Some complex tasks might involve a **flow** that coordinates multiple agents. 15 | It keeps track of the conversation using `Memory` and ensures secure code execution using a `DockerSandbox`. 16 | The system is flexible, allowing new tools to be added, even dynamically through the `MCP` protocol. 17 | 18 | ```mermaid 19 | flowchart TD 20 | A0["BaseAgent"] 21 | A1["Tool / ToolCollection"] 22 | A2["LLM"] 23 | A3["Message / Memory"] 24 | A4["Schema"] 25 | A5["BaseFlow"] 26 | A6["DockerSandbox"] 27 | A7["Configuration (Config)"] 28 | A8["MCP (Model Context Protocol)"] 29 | A0 -- "Uses LLM for thinking" --> A2 30 | A0 -- "Uses Memory for context" --> A3 31 | A0 -- "Executes Tools" --> A1 32 | A5 -- "Orchestrates Agents" --> A0 33 | A1 -- "Uses Sandbox for execution" --> A6 34 | A2 -- "Reads LLM Config" --> A7 35 | A6 -- "Reads Sandbox Config" --> A7 36 | A7 -- "Provides MCP Config" --> A8 37 | A8 -- "Provides Dynamic Tools" --> A1 38 | A8 -- "Extends BaseAgent" --> A0 39 | A4 -- "Defines Agent Structures" --> A0 40 | A4 -- "Defines Message Structure" --> A3 41 | A2 -- "Processes Messages" --> A3 42 | A5 -- "Uses Tools" --> A1 43 | A4 -- "Defines Tool Structures" --> A1 44 | ``` 45 | -------------------------------------------------------------------------------- /docs/PocketFlow/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "PocketFlow" 4 | nav_order: 18 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: PocketFlow 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | PocketFlow[View Repo](https://github.com/The-Pocket/PocketFlow) is a *Python framework* for building modular workflows and AI agents. 13 | It allows you to define complex processes by connecting individual **Nodes**, which represent *atomic tasks* like calling an LLM or searching the web. 14 | A **Flow** then *orchestrates* these Nodes, guiding the execution sequence based on **Actions** (string identifiers) returned by each Node. 15 | Data is passed between Nodes and managed throughout the workflow execution via a **Shared State** (a Python dictionary). 16 | PocketFlow also offers advanced features like **Batch Processing** for efficiently handling collections of items, and **Asynchronous Processing** for non-blocking operations crucial for I/O-bound tasks. 17 | Additionally, it demonstrates an **A2A (Agent-to-Agent) Communication Framework** to wrap PocketFlow agents, enabling them to communicate with other systems using a standardized JSON-RPC protocol. 18 | 19 | ```mermaid 20 | flowchart TD 21 | A0["Node (BaseNode, Node, AsyncNode) 22 | "] 23 | A1["Flow (Flow, AsyncFlow) 24 | "] 25 | A2["Shared State (shared dictionary) 26 | "] 27 | A3["Actions / Transitions 28 | "] 29 | A4["Batch Processing (BatchNode, BatchFlow, AsyncParallelBatchNode) 30 | "] 31 | A5["Asynchronous Processing (AsyncNode, AsyncFlow) 32 | "] 33 | A6["A2A (Agent-to-Agent) Communication Framework 34 | "] 35 | A1 -- "Orchestrates Nodes" --> A0 36 | A0 -- "Accesses Shared State" --> A2 37 | A0 -- "Returns Action" --> A3 38 | A1 -- "Uses Action for dispatch" --> A3 39 | A4 -- "Specializes Node (batch)" --> A0 40 | A4 -- "Specializes Flow (batch)" --> A1 41 | A5 -- "Specializes Node (async)" --> A0 42 | A5 -- "Specializes Flow (async)" --> A1 43 | A6 -- "Executes Flow" --> A1 44 | A6 -- "Initializes Shared State" --> A2 45 | ``` 46 | 47 | --- 48 | 49 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/Pydantic Core/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Pydantic Core" 4 | nav_order: 18 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: Pydantic Core 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | Pydantic Core[View Repo](https://github.com/pydantic/pydantic/tree/6c38dc93f40a47f4d1350adca9ec0d72502e223f/pydantic) provides the fundamental machinery for **data validation**, **parsing**, and **serialization** in Pydantic. It takes Python *type hints* and uses them to define how data should be structured and processed. Users typically interact with it by defining classes that inherit from `BaseModel`, which automatically gets validation and serialization capabilities based on its annotated fields. Pydantic Core ensures data conforms to the defined types and allows converting between Python objects and formats like JSON efficiently, leveraging Rust for performance. 13 | 14 | ```mermaid 15 | flowchart TD 16 | A0["BaseModel"] 17 | A1["Fields (FieldInfo / Field function)"] 18 | A2["Core Schema & Validation/Serialization"] 19 | A3["Configuration (ConfigDict / ConfigWrapper)"] 20 | A4["Custom Logic (Decorators & Annotated Helpers)"] 21 | A5["TypeAdapter"] 22 | A0 -- "Contains and defines" --> A1 23 | A0 -- "Is configured by" --> A3 24 | A0 -- "Applies custom logic via" --> A4 25 | A1 -- "Is converted into" --> A2 26 | A3 -- "Configures core engine for" --> A2 27 | A4 -- "Modifies validation/seriali..." --> A2 28 | A5 -- "Uses core engine for" --> A2 29 | A5 -- "Can be configured by" --> A3 30 | ``` -------------------------------------------------------------------------------- /docs/Requests/01_functional_api.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Functional API" 4 | parent: "Requests" 5 | nav_order: 1 6 | --- 7 | 8 | # Chapter 1: The Simplest Way - The Functional API 9 | 10 | Welcome to the world of `Requests`! If you need to get information from a website or interact with a web service using Python, `Requests` is your friendly helper. 11 | 12 | Imagine you just want to quickly grab the content of a webpage, maybe check the latest news headlines from a site, or send a simple piece of data to an online service. How do you do that without getting bogged down in complex details? 13 | 14 | That's where the **Functional API** of `Requests` comes in. It's the most straightforward way to start making web requests. 15 | 16 | ## What's the Functional API? 17 | 18 | Think of the Functional API as a set of handy, ready-to-use tools right at the top level of the `requests` library. You don't need to set anything up; you just call a function like `requests.get()` to fetch data or `requests.post()` to send data. 19 | 20 | **Analogy:** Ordering Takeout 🍕 21 | 22 | Using the Functional API is like using a generic food delivery app (like DoorDash or Uber Eats) to order a pizza from a place you've never ordered from before. 23 | 24 | 1. You open the app ( `import requests`). 25 | 2. You find the pizza place and tap "Order" (`requests.get('pizza_place_url')`). 26 | 3. The app handles finding a driver, sending them to the restaurant, picking up the pizza, and delivering it to you (Requests does all the connection and fetching work). 27 | 4. You get your pizza (`Response` object). 28 | 29 | It's super convenient for a one-time order! 30 | 31 | ## Making Your First Request: `requests.get()` 32 | 33 | The most common type of request is a `GET` request. It's what your web browser does every time you type a website address and hit Enter. It means "Please *get* me the content of this page." 34 | 35 | Let's try it! First, make sure you have `requests` installed (`pip install requests`). Then, in your Python script or interactive session: 36 | 37 | ```python 38 | import requests # Import the library 39 | 40 | # The URL we want to get data from 41 | url = 'https://httpbin.org/get' # A handy website for testing requests 42 | 43 | # Use the functional API 'get' function 44 | print(f"Fetching data from: {url}") 45 | response = requests.get(url) 46 | 47 | # Check if the request was successful (Status Code 200 means OK) 48 | print(f"Status Code: {response.status_code}") 49 | 50 | # Print the first 200 characters of the content we received 51 | print("Response Content (first 200 chars):") 52 | print(response.text[:200]) 53 | ``` 54 | 55 | **What happened here?** 56 | 57 | 1. `import requests`: We told Python we want to use the `requests` library. 58 | 2. `response = requests.get(url)`: This is the core magic! We called the `get` function directly from the `requests` module, passing the URL we want to visit. 59 | 3. `requests` did all the work: connected to the server, sent the `GET` request, and received the server's reply. 60 | 4. The reply is stored in the `response` variable. This isn't just the text of the page; it's a special `Response` object containing lots of useful information. We'll explore this more in [Request & Response Models](02_request___response_models.md). 61 | 5. `response.status_code`: We checked the status code. `200` is the standard code for "Everything went okay!". Other codes might indicate errors (like `404 Not Found`). 62 | 6. `response.text`: We accessed the main content (usually HTML or JSON) returned by the server as a string. 63 | 64 | ## Sending Data: `requests.post()` 65 | 66 | Sometimes, instead of just getting data, you need to *send* data to a website. This is often done when submitting a form, logging in, or telling an API to perform an action. The `POST` method is commonly used for this. 67 | 68 | The Functional API provides `requests.post()` for this purpose. 69 | 70 | ```python 71 | import requests 72 | 73 | # The URL we want to send data to 74 | url = 'https://httpbin.org/post' 75 | 76 | # The data we want to send (like form fields) 77 | # We'll use a Python dictionary 78 | payload = {'username': 'tutorial_user', 'action': 'learn_requests'} 79 | 80 | print(f"Sending data to: {url}") 81 | # Use the functional API 'post' function, passing the data 82 | response = requests.post(url, data=payload) 83 | 84 | # Check the status code 85 | print(f"Status Code: {response.status_code}") 86 | 87 | # The response often echoes back the data we sent 88 | print("Response Content:") 89 | print(response.text) 90 | ``` 91 | 92 | **What's new?** 93 | 94 | 1. `payload = {...}`: We created a Python dictionary to hold the data we want to send. 95 | 2. `response = requests.post(url, data=payload)`: We called `requests.post()`. Notice the second argument, `data=payload`. This tells `requests` to send our dictionary as form data in the body of the `POST` request. 96 | 3. The `response.text` from `httpbin.org/post` conveniently shows us the data it received, confirming our `payload` was sent correctly. 97 | 98 | `Requests` also offers functions for other HTTP methods like `put`, `delete`, `head`, `patch`, and `options`, all working similarly: `requests.put(...)`, `requests.delete(...)`, etc. 99 | 100 | ## How It Works Under the Hood 101 | 102 | You might wonder: if it's so simple, how does `requests.get()` actually connect to the internet and manage the request? 103 | 104 | Every time you call one of these functional API methods (like `requests.get` or `requests.post`), `Requests` performs a few steps behind the scenes: 105 | 106 | 1. **Creates a temporary `Session` object:** Think of a `Session` as a more advanced way to manage requests, especially when you need to talk to the same website multiple times. We'll learn all about these in the [Session](03_session.md) chapter. For a functional API call, `requests` creates a *brand new, temporary* `Session` just for this single request. 107 | 2. **Uses the `Session`:** This temporary `Session` is then used to actually prepare and send your request (e.g., the `GET` to `https://httpbin.org/get`). 108 | 3. **Gets the `Response`:** The `Session` receives the reply from the server. 109 | 4. **Returns the `Response` to you:** The function gives you back the `Response` object. 110 | 5. **Discards the `Session`:** The temporary `Session` is immediately thrown away. It's gone. 111 | 112 | **Analogy Revisited:** The generic delivery app (Functional API) contacts *a* driver (creates a temporary `Session`), tells them the restaurant and your order (sends the request), the driver delivers the food (returns the `Response`), and then the app forgets about that specific driver (discards the `Session`). If you order again 5 minutes later, it starts the whole process over with potentially a different driver. 113 | 114 | Here's a simplified diagram of what happens when you call `requests.get()`: 115 | 116 | ```mermaid 117 | sequenceDiagram 118 | participant User as Your Code 119 | participant FuncAPI as requests.get() 120 | participant TempSession as Temporary Session 121 | participant Server as Web Server 122 | 123 | User->>FuncAPI: Call requests.get('url') 124 | FuncAPI->>TempSession: Create new Session() 125 | activate TempSession 126 | TempSession->>Server: Make HTTP GET request to 'url' 127 | activate Server 128 | Server-->>TempSession: Send HTTP Response back 129 | deactivate Server 130 | TempSession-->>FuncAPI: Return Response object 131 | FuncAPI-->>User: Return Response object 132 | deactivate TempSession 133 | Note right of FuncAPI: Temporary Session is discarded 134 | ``` 135 | 136 | You can see a glimpse of this in the `requests/api.py` code: 137 | 138 | ```python 139 | # File: requests/api.py (Simplified view) 140 | 141 | from . import sessions # Where the Session logic lives 142 | 143 | def request(method, url, **kwargs): 144 | """Internal function that handles all functional API calls.""" 145 | 146 | # Creates a temporary Session just for this one call. 147 | # The 'with' statement ensures it's properly closed afterwards. 148 | with sessions.Session() as session: 149 | # The temporary session makes the actual request. 150 | return session.request(method=method, url=url, **kwargs) 151 | 152 | def get(url, params=None, **kwargs): 153 | """Sends a GET request (functional API).""" 154 | # This is just a convenient shortcut that calls the main 'request' function. 155 | return request("get", url, params=params, **kwargs) 156 | 157 | def post(url, data=None, json=None, **kwargs): 158 | """Sends a POST request (functional API).""" 159 | # Another shortcut calling the main 'request' function. 160 | return request("post", url, data=data, json=json, **kwargs) 161 | 162 | # ... similar functions for put, delete, head, patch, options ... 163 | ``` 164 | 165 | Each function like `get`, `post`, etc., is just a simple wrapper that calls the main `request` function, which in turn creates and uses that temporary `Session`. 166 | 167 | ## When Is It Good? When Is It Not? 168 | 169 | **Good For:** 170 | 171 | * Simple, one-off requests. 172 | * Quick scripts where performance isn't critical. 173 | * Learning `Requests` - it's the easiest starting point! 174 | 175 | **Not Ideal For:** 176 | 177 | * **Multiple requests to the same website:** Creating and tearing down a connection and a `Session` for *every single request* is inefficient. It's like sending a separate delivery driver for each item you forgot from the grocery store. 178 | * **Needing persistence:** If the website gives you a cookie (like after logging in) and you want to use it on your *next* request to that same site, the functional API won't remember it because the temporary `Session` (which holds cookies) is discarded after each call. 179 | * **Fine-grained control:** If you need custom configurations, specific connection pooling, or advanced features, using a `Session` object directly offers more power. 180 | 181 | ## Conclusion 182 | 183 | You've learned about the `Requests` Functional API – the simplest way to make web requests using functions like `requests.get()` and `requests.post()`. It's perfect for quick tasks and getting started. You saw how it works by creating temporary `Session` objects behind the scenes. 184 | 185 | While convenient for single shots, remember its limitations for performance and state persistence when dealing with multiple requests to the same site. 186 | 187 | Now that you know how to *send* a basic request, what exactly do you get *back*? Let's explore the structure of the requests we send and the powerful `Response` object we receive. 188 | 189 | **Next:** [Chapter 2: Request & Response Models](02_request___response_models.md) 190 | 191 | --- 192 | 193 | Generated by [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) -------------------------------------------------------------------------------- /docs/Requests/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Requests" 4 | nav_order: 19 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: Requests 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | Requests[View Repo](https://github.com/psf/requests/tree/0e322af87745eff34caffe4df68456ebc20d9068/src/requests) is a Python library that makes sending *HTTP requests* incredibly simple. 13 | Instead of dealing with complex details, you can use straightforward functions (like `requests.get()`) or **Session objects** to interact with web services. 14 | It automatically handles things like *cookies*, *redirects*, *authentication*, and connection pooling, returning easy-to-use **Response objects** with all the server's data. 15 | 16 | ```mermaid 17 | flowchart TD 18 | A0["Request & Response Models"] 19 | A1["Session"] 20 | A2["Transport Adapters"] 21 | A3["Functional API"] 22 | A4["Authentication Handlers"] 23 | A5["Cookie Jar"] 24 | A6["Exception Hierarchy"] 25 | A7["Hook System"] 26 | A3 -- "Uses temporary" --> A1 27 | A1 -- "Prepares/Receives" --> A0 28 | A1 -- "Manages & Uses" --> A2 29 | A1 -- "Manages" --> A5 30 | A1 -- "Manages" --> A4 31 | A1 -- "Manages" --> A7 32 | A2 -- "Sends/Builds" --> A0 33 | A4 -- "Modifies (adds headers)" --> A0 34 | A5 -- "Populates/Reads" --> A0 35 | A7 -- "Operates on" --> A0 36 | A0 -- "Can Raise (raise_for_status)" --> A6 37 | A2 -- "Raises Connection Errors" --> A6 38 | ``` -------------------------------------------------------------------------------- /docs/SmolaAgents/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "SmolaAgents" 4 | nav_order: 20 5 | has_children: true 6 | --- 7 | 8 | # Tutorial: SmolaAgents 9 | 10 | > This tutorial is AI-generated! To learn more, check out [AI Codebase Knowledge Builder](https://github.com/The-Pocket/Tutorial-Codebase-Knowledge) 11 | 12 | `SmolaAgents`[View Repo](https://github.com/huggingface/smolagents/tree/076cca5e8a130d3fa2ff990ad630231b49767745/src/smolagents) is a project for building *autonomous agents* that can solve complex tasks. 13 | The core component is the **MultiStepAgent**, which acts like a project manager. It uses a **Model Interface** to talk to language models (LLMs), employs **Tools** (like web search or code execution) to interact with the world or perform actions, and keeps track of its progress and conversation history using **AgentMemory**. 14 | For agents that write and run Python code (`CodeAgent`), a **PythonExecutor** provides a safe environment. **PromptTemplates** help structure the instructions given to the LLM, while **AgentType** handles different data formats like images or audio. Finally, **AgentLogger & Monitor** provides logging and tracking for debugging and analysis. 15 | 16 | ```mermaid 17 | flowchart TD 18 | A0["MultiStepAgent"] 19 | A1["Tool"] 20 | A2["Model Interface"] 21 | A3["AgentMemory"] 22 | A4["PythonExecutor"] 23 | A5["PromptTemplates"] 24 | A6["AgentType"] 25 | A7["AgentLogger & Monitor"] 26 | A0 -- "Uses tools" --> A1 27 | A0 -- "Uses model" --> A2 28 | A0 -- "Uses memory" --> A3 29 | A0 -- "Uses templates" --> A5 30 | A0 -- "Uses logger/monitor" --> A7 31 | A0 -- "Uses executor (CodeAgent)" --> A4 32 | A1 -- "Outputs agent types" --> A6 33 | A4 -- "Executes tool code" --> A1 34 | A2 -- "Generates/Parses tool calls" --> A1 35 | A3 -- "Logs tool calls" --> A1 36 | A5 -- "Includes tool info" --> A1 37 | A6 -- "Handled by agent" --> A0 38 | A7 -- "Replays memory" --> A3 39 | ``` -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | # Basic site settings 2 | title: Pocket Flow 3 | 4 | # Theme settings 5 | remote_theme: just-the-docs/just-the-docs 6 | 7 | # Navigation 8 | nav_sort: case_sensitive 9 | 10 | # Aux links (shown in upper right) 11 | aux_links: 12 | "View on GitHub": 13 | - "https://github.com/the-pocket/Tutorial-Codebase-Knowledge" 14 | 15 | # Color scheme 16 | color_scheme: light 17 | 18 | # Author settings 19 | author: 20 | name: Zachary Huang 21 | url: https://www.columbia.edu/~zh2408/ 22 | twitter: ZacharyHuang12 23 | 24 | # Mermaid settings 25 | mermaid: 26 | version: "11.6.0" # Pick the version you want 27 | # Default configuration 28 | config: | 29 | directionLR 30 | 31 | # Callouts settings 32 | callouts: 33 | warning: 34 | title: Warning 35 | color: red 36 | note: 37 | title: Note 38 | color: blue 39 | best-practice: 40 | title: Best Practice 41 | color: green 42 | 43 | # The custom navigation 44 | nav: 45 | - Home: index.md # Link to your main docs index 46 | - GitHub: "https://github.com/The-Pocket/Tutorial-Codebase-Knowledge" 47 | - Discord: "https://discord.gg/hUHHE9Sa6T" 48 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: default 3 | title: "Home" 4 | nav_order: 1 5 | --- 6 | 7 | # Turns Codebase into Easy Tutorial - Pocket Flow 8 | 9 | Ever stared at a new codebase written by others feeling completely lost? This project analyzes GitHub repositories and creates beginner-friendly tutorials explaining exactly how the code works - all powered by AI! Our intelligent system automatically breaks down complex codebases into digestible explanations that even beginners can understand. 10 | 11 |

12 | 13 | 16 | 17 |

18 | 19 | This is a tutorial project of [Pocket Flow](https://github.com/The-Pocket/PocketFlow), a 100-line LLM framework. It crawls GitHub repositories and build a knowledge base from the code. 20 | 21 | ## Example Tutorials for Popular GitHub Repositories 22 | 23 | - [AutoGen Core](./AutoGen Core/index.md) - Build AI teams that talk, think, and solve problems together like coworkers! 24 | - [Browser Use](./Browser Use/index.md) - Let AI surf the web for you, clicking buttons and filling forms like a digital assistant! 25 | - [Celery](./Celery/index.md) - Supercharge your app with background tasks that run while you sleep! 26 | - [Click](./Click/index.md) - Turn Python functions into slick command-line tools with just a decorator! 27 | - [Codex](./Codex/index.md) - Turn plain English into working code with this AI terminal wizard! 28 | - [Crawl4AI](./Crawl4AI/index.md) - Train your AI to extract exactly what matters from any website! 29 | - [CrewAI](./CrewAI/index.md) - Assemble a dream team of AI specialists to tackle impossible problems! 30 | - [DSPy](./DSPy/index.md) - Build LLM apps like Lego blocks that optimize themselves! 31 | - [FastAPI](./FastAPI/index.md) - Create APIs at lightning speed with automatic docs that clients will love! 32 | - [Flask](./Flask/index.md) - Craft web apps with minimal code that scales from prototype to production! 33 | - [Google A2A](./Google A2A/index.md) - The universal language that lets AI agents collaborate across borders! 34 | - [LangGraph](./LangGraph/index.md) - Design AI agents as flowcharts where each step remembers what happened before! 35 | - [LevelDB](./LevelDB/index.md) - Store data at warp speed with Google's engine that powers blockchains! 36 | - [MCP Python SDK](./MCP Python SDK/index.md) - Build powerful apps that communicate through an elegant protocol without sweating the details! 37 | - [NumPy Core](./NumPy Core/index.md) - Master the engine behind data science that makes Python as fast as C! 38 | - [OpenManus](./OpenManus/index.md) - Build AI agents with digital brains that think, learn, and use tools just like humans do! 39 | - [PocketFlow](./PocketFlow/index.md) - 100-line LLM framework. Let Agents build Agents! 40 | - [Pydantic Core](./Pydantic Core/index.md) - Validate data at rocket speed with just Python type hints! 41 | - [Requests](./Requests/index.md) - Talk to the internet in Python with code so simple it feels like cheating! 42 | - [SmolaAgents](./SmolaAgents/index.md) - Build tiny AI agents that punch way above their weight class! 43 | 44 | -------------------------------------------------------------------------------- /flow.py: -------------------------------------------------------------------------------- 1 | from pocketflow import Flow 2 | # Import all node classes from nodes.py 3 | from nodes import ( 4 | FetchRepo, 5 | IdentifyAbstractions, 6 | AnalyzeRelationships, 7 | OrderChapters, 8 | WriteChapters, 9 | CombineTutorial 10 | ) 11 | 12 | def create_tutorial_flow(): 13 | """Creates and returns the codebase tutorial generation flow.""" 14 | 15 | # Instantiate nodes 16 | fetch_repo = FetchRepo() 17 | identify_abstractions = IdentifyAbstractions(max_retries=5, wait=20) 18 | analyze_relationships = AnalyzeRelationships(max_retries=5, wait=20) 19 | order_chapters = OrderChapters(max_retries=5, wait=20) 20 | write_chapters = WriteChapters(max_retries=5, wait=20) # This is a BatchNode 21 | combine_tutorial = CombineTutorial() 22 | 23 | # Connect nodes in sequence based on the design 24 | fetch_repo >> identify_abstractions 25 | identify_abstractions >> analyze_relationships 26 | analyze_relationships >> order_chapters 27 | order_chapters >> write_chapters 28 | write_chapters >> combine_tutorial 29 | 30 | # Create the flow starting with FetchRepo 31 | tutorial_flow = Flow(start=fetch_repo) 32 | 33 | return tutorial_flow 34 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import dotenv 2 | import os 3 | import argparse 4 | # Import the function that creates the flow 5 | from flow import create_tutorial_flow 6 | 7 | dotenv.load_dotenv() 8 | 9 | # Default file patterns 10 | DEFAULT_INCLUDE_PATTERNS = { 11 | "*.py", "*.js", "*.jsx", "*.ts", "*.tsx", "*.go", "*.java", "*.pyi", "*.pyx", 12 | "*.c", "*.cc", "*.cpp", "*.h", "*.md", "*.rst", "*Dockerfile", 13 | "*Makefile", "*.yaml", "*.yml", 14 | } 15 | 16 | DEFAULT_EXCLUDE_PATTERNS = { 17 | "assets/*", "data/*", "images/*", "public/*", "static/*", "temp/*", 18 | "*docs/*", 19 | "*venv/*", 20 | "*.venv/*", 21 | "*test*", 22 | "*tests/*", 23 | "*examples/*", 24 | "v1/*", 25 | "*dist/*", 26 | "*build/*", 27 | "*experimental/*", 28 | "*deprecated/*", 29 | "*misc/*", 30 | "*legacy/*", 31 | ".git/*", ".github/*", ".next/*", ".vscode/*", 32 | "*obj/*", 33 | "*bin/*", 34 | "*node_modules/*", 35 | "*.log" 36 | } 37 | 38 | # --- Main Function --- 39 | def main(): 40 | parser = argparse.ArgumentParser(description="Generate a tutorial for a GitHub codebase or local directory.") 41 | 42 | # Create mutually exclusive group for source 43 | source_group = parser.add_mutually_exclusive_group(required=True) 44 | source_group.add_argument("--repo", help="URL of the public GitHub repository.") 45 | source_group.add_argument("--dir", help="Path to local directory.") 46 | 47 | parser.add_argument("-n", "--name", help="Project name (optional, derived from repo/directory if omitted).") 48 | parser.add_argument("-t", "--token", help="GitHub personal access token (optional, reads from GITHUB_TOKEN env var if not provided).") 49 | parser.add_argument("-o", "--output", default="output", help="Base directory for output (default: ./output).") 50 | parser.add_argument("-i", "--include", nargs="+", help="Include file patterns (e.g. '*.py' '*.js'). Defaults to common code files if not specified.") 51 | parser.add_argument("-e", "--exclude", nargs="+", help="Exclude file patterns (e.g. 'tests/*' 'docs/*'). Defaults to test/build directories if not specified.") 52 | parser.add_argument("-s", "--max-size", type=int, default=100000, help="Maximum file size in bytes (default: 100000, about 100KB).") 53 | # Add language parameter for multi-language support 54 | parser.add_argument("--language", default="english", help="Language for the generated tutorial (default: english)") 55 | # Add use_cache parameter to control LLM caching 56 | parser.add_argument("--no-cache", action="store_true", help="Disable LLM response caching (default: caching enabled)") 57 | # Add max_abstraction_num parameter to control the number of abstractions 58 | parser.add_argument("--max-abstractions", type=int, default=10, help="Maximum number of abstractions to identify (default: 10)") 59 | 60 | args = parser.parse_args() 61 | 62 | # Get GitHub token from argument or environment variable if using repo 63 | github_token = None 64 | if args.repo: 65 | github_token = args.token or os.environ.get('GITHUB_TOKEN') 66 | if not github_token: 67 | print("Warning: No GitHub token provided. You might hit rate limits for public repositories.") 68 | 69 | # Initialize the shared dictionary with inputs 70 | shared = { 71 | "repo_url": args.repo, 72 | "local_dir": args.dir, 73 | "project_name": args.name, # Can be None, FetchRepo will derive it 74 | "github_token": github_token, 75 | "output_dir": args.output, # Base directory for CombineTutorial output 76 | 77 | # Add include/exclude patterns and max file size 78 | "include_patterns": set(args.include) if args.include else DEFAULT_INCLUDE_PATTERNS, 79 | "exclude_patterns": set(args.exclude) if args.exclude else DEFAULT_EXCLUDE_PATTERNS, 80 | "max_file_size": args.max_size, 81 | 82 | # Add language for multi-language support 83 | "language": args.language, 84 | 85 | # Add use_cache flag (inverse of no-cache flag) 86 | "use_cache": not args.no_cache, 87 | 88 | # Add max_abstraction_num parameter 89 | "max_abstraction_num": args.max_abstractions, 90 | 91 | # Outputs will be populated by the nodes 92 | "files": [], 93 | "abstractions": [], 94 | "relationships": {}, 95 | "chapter_order": [], 96 | "chapters": [], 97 | "final_output_dir": None 98 | } 99 | 100 | # Display starting message with repository/directory and language 101 | print(f"Starting tutorial generation for: {args.repo or args.dir} in {args.language.capitalize()} language") 102 | print(f"LLM caching: {'Disabled' if args.no_cache else 'Enabled'}") 103 | 104 | # Create the flow instance 105 | tutorial_flow = create_tutorial_flow() 106 | 107 | # Run the flow 108 | tutorial_flow.run(shared) 109 | 110 | if __name__ == "__main__": 111 | main() 112 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pocketflow>=0.0.1 2 | pyyaml>=6.0 3 | requests>=2.28.0 4 | gitpython>=3.1.0 5 | google-cloud-aiplatform>=1.25.0 6 | google-genai>=1.9.0 7 | python-dotenv>=1.0.0 8 | pathspec>=0.11.0 9 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/The-Pocket/PocketFlow-Tutorial-Codebase-Knowledge/755e76733dfd257997bc039727697c8000b38aa6/utils/__init__.py -------------------------------------------------------------------------------- /utils/call_llm.py: -------------------------------------------------------------------------------- 1 | from google import genai 2 | import os 3 | import logging 4 | import json 5 | from datetime import datetime 6 | 7 | # Configure logging 8 | log_directory = os.getenv("LOG_DIR", "logs") 9 | os.makedirs(log_directory, exist_ok=True) 10 | log_file = os.path.join( 11 | log_directory, f"llm_calls_{datetime.now().strftime('%Y%m%d')}.log" 12 | ) 13 | 14 | # Set up logger 15 | logger = logging.getLogger("llm_logger") 16 | logger.setLevel(logging.INFO) 17 | logger.propagate = False # Prevent propagation to root logger 18 | file_handler = logging.FileHandler(log_file, encoding='utf-8') 19 | file_handler.setFormatter( 20 | logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") 21 | ) 22 | logger.addHandler(file_handler) 23 | 24 | # Simple cache configuration 25 | cache_file = "llm_cache.json" 26 | 27 | 28 | # By default, we Google Gemini 2.5 pro, as it shows great performance for code understanding 29 | def call_llm(prompt: str, use_cache: bool = True) -> str: 30 | # Log the prompt 31 | logger.info(f"PROMPT: {prompt}") 32 | 33 | # Check cache if enabled 34 | if use_cache: 35 | # Load cache from disk 36 | cache = {} 37 | if os.path.exists(cache_file): 38 | try: 39 | with open(cache_file, "r", encoding="utf-8") as f: 40 | cache = json.load(f) 41 | except: 42 | logger.warning(f"Failed to load cache, starting with empty cache") 43 | 44 | # Return from cache if exists 45 | if prompt in cache: 46 | logger.info(f"RESPONSE: {cache[prompt]}") 47 | return cache[prompt] 48 | 49 | # # Call the LLM if not in cache or cache disabled 50 | # client = genai.Client( 51 | # vertexai=True, 52 | # # TODO: change to your own project id and location 53 | # project=os.getenv("GEMINI_PROJECT_ID", "your-project-id"), 54 | # location=os.getenv("GEMINI_LOCATION", "us-central1") 55 | # ) 56 | 57 | # You can comment the previous line and use the AI Studio key instead: 58 | client = genai.Client( 59 | api_key=os.getenv("GEMINI_API_KEY", ""), 60 | ) 61 | model = os.getenv("GEMINI_MODEL", "gemini-2.5-pro-exp-03-25") 62 | # model = os.getenv("GEMINI_MODEL", "gemini-2.5-flash-preview-04-17") 63 | 64 | response = client.models.generate_content(model=model, contents=[prompt]) 65 | response_text = response.text 66 | 67 | # Log the response 68 | logger.info(f"RESPONSE: {response_text}") 69 | 70 | # Update cache if enabled 71 | if use_cache: 72 | # Load cache again to avoid overwrites 73 | cache = {} 74 | if os.path.exists(cache_file): 75 | try: 76 | with open(cache_file, "r", encoding="utf-8") as f: 77 | cache = json.load(f) 78 | except: 79 | pass 80 | 81 | # Add to cache and save 82 | cache[prompt] = response_text 83 | try: 84 | with open(cache_file, "w", encoding="utf-8") as f: 85 | json.dump(cache, f) 86 | except Exception as e: 87 | logger.error(f"Failed to save cache: {e}") 88 | 89 | return response_text 90 | 91 | 92 | # # Use Azure OpenAI 93 | # def call_llm(prompt, use_cache: bool = True): 94 | # from openai import AzureOpenAI 95 | 96 | # endpoint = "https://.openai.azure.com/" 97 | # deployment = "" 98 | 99 | # subscription_key = "" 100 | # api_version = "" 101 | 102 | # client = AzureOpenAI( 103 | # api_version=api_version, 104 | # azure_endpoint=endpoint, 105 | # api_key=subscription_key, 106 | # ) 107 | 108 | # r = client.chat.completions.create( 109 | # model=deployment, 110 | # messages=[{"role": "user", "content": prompt}], 111 | # response_format={ 112 | # "type": "text" 113 | # }, 114 | # max_completion_tokens=40000, 115 | # reasoning_effort="medium", 116 | # store=False 117 | # ) 118 | # return r.choices[0].message.content 119 | 120 | # # Use Anthropic Claude 3.7 Sonnet Extended Thinking 121 | # def call_llm(prompt, use_cache: bool = True): 122 | # from anthropic import Anthropic 123 | # client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY", "your-api-key")) 124 | # response = client.messages.create( 125 | # model="claude-3-7-sonnet-20250219", 126 | # max_tokens=21000, 127 | # thinking={ 128 | # "type": "enabled", 129 | # "budget_tokens": 20000 130 | # }, 131 | # messages=[ 132 | # {"role": "user", "content": prompt} 133 | # ] 134 | # ) 135 | # return response.content[1].text 136 | 137 | # # Use OpenAI o1 138 | # def call_llm(prompt, use_cache: bool = True): 139 | # from openai import OpenAI 140 | # client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "your-api-key")) 141 | # r = client.chat.completions.create( 142 | # model="o1", 143 | # messages=[{"role": "user", "content": prompt}], 144 | # response_format={ 145 | # "type": "text" 146 | # }, 147 | # reasoning_effort="medium", 148 | # store=False 149 | # ) 150 | # return r.choices[0].message.content 151 | 152 | # Use OpenRouter API 153 | # def call_llm(prompt: str, use_cache: bool = True) -> str: 154 | # import requests 155 | # # Log the prompt 156 | # logger.info(f"PROMPT: {prompt}") 157 | 158 | # # Check cache if enabled 159 | # if use_cache: 160 | # # Load cache from disk 161 | # cache = {} 162 | # if os.path.exists(cache_file): 163 | # try: 164 | # with open(cache_file, "r", encoding="utf-8") as f: 165 | # cache = json.load(f) 166 | # except: 167 | # logger.warning(f"Failed to load cache, starting with empty cache") 168 | 169 | # # Return from cache if exists 170 | # if prompt in cache: 171 | # logger.info(f"RESPONSE: {cache[prompt]}") 172 | # return cache[prompt] 173 | 174 | # # OpenRouter API configuration 175 | # api_key = os.getenv("OPENROUTER_API_KEY", "") 176 | # model = os.getenv("OPENROUTER_MODEL", "google/gemini-2.0-flash-exp:free") 177 | 178 | # headers = { 179 | # "Authorization": f"Bearer {api_key}", 180 | # } 181 | 182 | # data = { 183 | # "model": model, 184 | # "messages": [{"role": "user", "content": prompt}] 185 | # } 186 | 187 | # response = requests.post( 188 | # "https://openrouter.ai/api/v1/chat/completions", 189 | # headers=headers, 190 | # json=data 191 | # ) 192 | 193 | # if response.status_code != 200: 194 | # error_msg = f"OpenRouter API call failed with status {response.status_code}: {response.text}" 195 | # logger.error(error_msg) 196 | # raise Exception(error_msg) 197 | # try: 198 | # response_text = response.json()["choices"][0]["message"]["content"] 199 | # except Exception as e: 200 | # error_msg = f"Failed to parse OpenRouter response: {e}; Response: {response.text}" 201 | # logger.error(error_msg) 202 | # raise Exception(error_msg) 203 | 204 | 205 | # # Log the response 206 | # logger.info(f"RESPONSE: {response_text}") 207 | 208 | # # Update cache if enabled 209 | # if use_cache: 210 | # # Load cache again to avoid overwrites 211 | # cache = {} 212 | # if os.path.exists(cache_file): 213 | # try: 214 | # with open(cache_file, "r", encoding="utf-8") as f: 215 | # cache = json.load(f) 216 | # except: 217 | # pass 218 | 219 | # # Add to cache and save 220 | # cache[prompt] = response_text 221 | # try: 222 | # with open(cache_file, "w", encoding="utf-8") as f: 223 | # json.dump(cache, f) 224 | # except Exception as e: 225 | # logger.error(f"Failed to save cache: {e}") 226 | 227 | # return response_text 228 | 229 | if __name__ == "__main__": 230 | test_prompt = "Hello, how are you?" 231 | 232 | # First call - should hit the API 233 | print("Making call...") 234 | response1 = call_llm(test_prompt, use_cache=False) 235 | print(f"Response: {response1}") 236 | -------------------------------------------------------------------------------- /utils/crawl_local_files.py: -------------------------------------------------------------------------------- 1 | import os 2 | import fnmatch 3 | import pathspec 4 | 5 | 6 | def crawl_local_files( 7 | directory, 8 | include_patterns=None, 9 | exclude_patterns=None, 10 | max_file_size=None, 11 | use_relative_paths=True, 12 | ): 13 | """ 14 | Crawl files in a local directory with similar interface as crawl_github_files. 15 | Args: 16 | directory (str): Path to local directory 17 | include_patterns (set): File patterns to include (e.g. {"*.py", "*.js"}) 18 | exclude_patterns (set): File patterns to exclude (e.g. {"tests/*"}) 19 | max_file_size (int): Maximum file size in bytes 20 | use_relative_paths (bool): Whether to use paths relative to directory 21 | 22 | Returns: 23 | dict: {"files": {filepath: content}} 24 | """ 25 | if not os.path.isdir(directory): 26 | raise ValueError(f"Directory does not exist: {directory}") 27 | 28 | files_dict = {} 29 | 30 | # --- Load .gitignore --- 31 | gitignore_path = os.path.join(directory, ".gitignore") 32 | gitignore_spec = None 33 | if os.path.exists(gitignore_path): 34 | try: 35 | with open(gitignore_path, "r", encoding="utf-8-sig") as f: 36 | gitignore_patterns = f.readlines() 37 | gitignore_spec = pathspec.PathSpec.from_lines("gitwildmatch", gitignore_patterns) 38 | print(f"Loaded .gitignore patterns from {gitignore_path}") 39 | except Exception as e: 40 | print(f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}") 41 | 42 | all_files = [] 43 | for root, dirs, files in os.walk(directory): 44 | # Filter directories using .gitignore and exclude_patterns early 45 | excluded_dirs = set() 46 | for d in dirs: 47 | dirpath_rel = os.path.relpath(os.path.join(root, d), directory) 48 | 49 | if gitignore_spec and gitignore_spec.match_file(dirpath_rel): 50 | excluded_dirs.add(d) 51 | continue 52 | 53 | if exclude_patterns: 54 | for pattern in exclude_patterns: 55 | if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(d, pattern): 56 | excluded_dirs.add(d) 57 | break 58 | 59 | for d in dirs.copy(): 60 | if d in excluded_dirs: 61 | dirs.remove(d) 62 | 63 | for filename in files: 64 | filepath = os.path.join(root, filename) 65 | all_files.append(filepath) 66 | 67 | total_files = len(all_files) 68 | processed_files = 0 69 | 70 | for filepath in all_files: 71 | relpath = os.path.relpath(filepath, directory) if use_relative_paths else filepath 72 | 73 | # --- Exclusion check --- 74 | excluded = False 75 | if gitignore_spec and gitignore_spec.match_file(relpath): 76 | excluded = True 77 | 78 | if not excluded and exclude_patterns: 79 | for pattern in exclude_patterns: 80 | if fnmatch.fnmatch(relpath, pattern): 81 | excluded = True 82 | break 83 | 84 | included = False 85 | if include_patterns: 86 | for pattern in include_patterns: 87 | if fnmatch.fnmatch(relpath, pattern): 88 | included = True 89 | break 90 | else: 91 | included = True 92 | 93 | processed_files += 1 # Increment processed count regardless of inclusion/exclusion 94 | 95 | status = "processed" 96 | if not included or excluded: 97 | status = "skipped (excluded)" 98 | # Print progress for skipped files due to exclusion 99 | if total_files > 0: 100 | percentage = (processed_files / total_files) * 100 101 | rounded_percentage = int(percentage) 102 | print(f"\033[92mProgress: {processed_files}/{total_files} ({rounded_percentage}%) {relpath} [{status}]\033[0m") 103 | continue # Skip to next file if not included or excluded 104 | 105 | if max_file_size and os.path.getsize(filepath) > max_file_size: 106 | status = "skipped (size limit)" 107 | # Print progress for skipped files due to size limit 108 | if total_files > 0: 109 | percentage = (processed_files / total_files) * 100 110 | rounded_percentage = int(percentage) 111 | print(f"\033[92mProgress: {processed_files}/{total_files} ({rounded_percentage}%) {relpath} [{status}]\033[0m") 112 | continue # Skip large files 113 | 114 | # --- File is being processed --- 115 | try: 116 | with open(filepath, "r", encoding="utf-8-sig") as f: 117 | content = f.read() 118 | files_dict[relpath] = content 119 | except Exception as e: 120 | print(f"Warning: Could not read file {filepath}: {e}") 121 | status = "skipped (read error)" 122 | 123 | # --- Print progress for processed or error files --- 124 | if total_files > 0: 125 | percentage = (processed_files / total_files) * 100 126 | rounded_percentage = int(percentage) 127 | print(f"\033[92mProgress: {processed_files}/{total_files} ({rounded_percentage}%) {relpath} [{status}]\033[0m") 128 | 129 | return {"files": files_dict} 130 | 131 | 132 | if __name__ == "__main__": 133 | print("--- Crawling parent directory ('..') ---") 134 | files_data = crawl_local_files( 135 | "..", 136 | exclude_patterns={ 137 | "*.pyc", 138 | "__pycache__/*", 139 | ".venv/*", 140 | ".git/*", 141 | "docs/*", 142 | "output/*", 143 | }, 144 | ) 145 | print(f"Found {len(files_data['files'])} files:") 146 | for path in files_data["files"]: 147 | print(f" {path}") --------------------------------------------------------------------------------