├── .env.example ├── .github ├── FUNDING.yml └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── LICENSE ├── README.md ├── assets └── img │ ├── pinecone.png │ └── rag-architecture.png ├── docs ├── [1]_sources.md ├── [2]_sources.md ├── [3]_sources.md ├── [4]_sources.md └── [5]_sources.md ├── full_basic_rag.ipynb ├── notebooks ├── [1]_rag_setup_overview.ipynb ├── [2]_rag_with_multi_query.ipynb ├── [3]_rag_routing_and_query_construction.ipynb ├── [4]_rag_indexing_and_advanced_retrieval.ipynb └── [5]_rag_retrieval_and_reranking.ipynb ├── requirements.txt └── test └── langchain_turing.pdf /.env.example: -------------------------------------------------------------------------------- 1 | # LLM Model - Get key at https://platform.openai.com/api-keys 2 | OPENAI_API_KEY=your-api-key 3 | 4 | # LangSmith - Get key at https://smith.langchain.com 5 | LANGCHAIN_TRACING_V2=true 6 | LANGCHAIN_ENDPOINT=https://api.smith.langchain.com 7 | LANGCHAIN_API_KEY=your-api-key 8 | LANGCHAIN_PROJECT=your-project-name 9 | 10 | # Pinecone Vector Database - Get key at https://app.pinecone.io 11 | PINECONE_INDEX_NAME=your-project-index 12 | PINECONE_API_HOST=your-host-url 13 | PINECONE_API_KEY=your-api-key 14 | 15 | # Cohere - Get key at https://dashboard.cohere.com/api-keys 16 | COHERE_API_KEY=your-api-key -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # bRAGAI 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 12 | polar: # Replace with a single Polar username 13 | buy_me_a_coffee: bragai 14 | thanks_dev: #gh/tahababou12 15 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 16 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | venv/ 3 | .ragatouille/ 4 | .DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 bRAG AI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | 1. The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | 2. If modifications or derivative works are made, proper credit must be given 16 | to the original project: **bRAG AI** (https://github.com/bragai/brag-langchain). 17 | This includes: 18 | - Clearly stating that the work is **derived from bRAG AI** in any documentation, 19 | readme files, or prominent places in the project. 20 | - Including a reference to the original repository. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER 26 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT, OR OTHERWISE, ARISING FROM, 27 | OUT OF, OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 28 | THE SOFTWARE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Retrieval-Augmented Generation (RAG) Project 2 | 3 | **_Think it. Build it. bRAG it._ 🚀 bRAGAI's coming soon (🤫)** 4 | 5 | **[Join the waitlist](https://bragai.dev/)** for exclusive early access, be among the first to try your AI-powered full-stack development assistant, and transform ideas into production-ready web apps in minutes. 6 | 7 | --------------------- 8 | 9 | This repository contains a comprehensive exploration of Retrieval-Augmented Generation (RAG) for various applications. 10 | Each notebook provides a detailed, hands-on guide to setting up and experimenting with RAG from an introductory level to advanced implementations, including multi-querying and custom RAG builds. 11 | 12 | ![rag_detail_v2](assets/img/rag-architecture.png) 13 | 14 | ## Project Structure 15 | 16 | If you want to jump straight into it, check out the file `full_basic_rag.ipynb` -> this file will give you a boilerplate starter code of a fully customizable RAG chatbot. 17 | 18 | Make sure to run your files in a virtual environment (checkout section `Get Started`) 19 | 20 | The following notebooks can be found under the directory `notebooks/`. 21 | 22 | ### [1]\_rag_setup_overview.ipynb 23 | 24 | This introductory notebook provides an overview of RAG architecture and its foundational setup. 25 | The notebook walks through: 26 | - **Environment Setup**: Configuring the environment, installing necessary libraries, and API setups. 27 | - **Initial Data Loading**: Basic document loaders and data preprocessing methods. 28 | - **Embedding Generation**: Generating embeddings using various models, including OpenAI's embeddings. 29 | - **Vector Store**: Setting up a vector store (ChromaDB/Pinecone) for efficient similarity search. 30 | - **Basic RAG Pipeline**: Creating a simple retrieval and generation pipeline to serve as a baseline. 31 | 32 | ### [2]\_rag_with_multi_query.ipynb 33 | 34 | Building on the basics, this notebook introduces multi-querying techniques in the RAG pipeline, exploring: 35 | - **Multi-Query Setup**: Configuring multiple queries to diversify retrieval. 36 | - **Advanced Embedding Techniques**: Utilizing multiple embedding models to refine retrieval. 37 | - **Pipeline with Multi-Querying**: Implementing multi-query handling to improve relevance in response generation. 38 | - **Comparison & Analysis**: Comparing results with single-query pipelines and analyzing performance improvements. 39 | 40 | ### [3]_rag_routing_and_query_construction.ipynb 41 | 42 | This notebook delves deeper into customizing a RAG pipeline. 43 | It covers: 44 | - **Logical Routing:** Implements function-based routing for classifying user queries to appropriate data sources based on programming languages. 45 | - **Semantic Routing:** Uses embeddings and cosine similarity to direct questions to either a math or physics prompt, optimizing response accuracy. 46 | - **Query Structuring for Metadata Filters:** Defines structured search schema for YouTube tutorial metadata, enabling advanced filtering (e.g., by view count, publication date). 47 | - **Structured Search Prompting:** Leverages LLM prompts to generate database queries for retrieving relevant content based on user input. 48 | - **Integration with Vector Stores:** Links structured queries to vector stores for efficient data retrieval. 49 | 50 | 51 | ### [4]_rag_indexing_and_advanced_retrieval.ipynb 52 | 53 | Continuing from the previous customization, this notebook explores: 54 | - **Preface on Document Chunking:** Points to external resources for document chunking techniques. 55 | - **Multi-representation Indexing:** Sets up a multi-vector indexing structure for handling documents with different embeddings and representations. 56 | - **In-Memory Storage for Summaries:** Uses InMemoryByteStore for storing document summaries alongside parent documents, enabling efficient retrieval. 57 | - **MultiVectorRetriever Setup:** Integrates multiple vector representations to retrieve relevant documents based on user queries. 58 | - **RAPTOR Implementation:** Explores RAPTOR, an advanced indexing and retrieval model, linking to in-depth resources. 59 | - **ColBERT Integration:** Demonstrates ColBERT-based token-level vector indexing and retrieval, which captures contextual meaning at a fine-grained level. 60 | - **Wikipedia Example with ColBERT:** Retrieves information about Hayao Miyazaki using the ColBERT retrieval model for demonstration. 61 | 62 | ### [5]_rag_retrieval_and_reranking.ipynb 63 | 64 | This final notebook brings together the RAG system components, with a focus on scalability and optimization: 65 | - **Document Loading and Splitting:** Loads and chunks documents for indexing, preparing them for vector storage. 66 | - **Multi-query Generation with RAG-Fusion:** Uses a prompt-based approach to generate multiple search queries from a single input question. 67 | - **Reciprocal Rank Fusion (RRF):** Implements RRF for re-ranking multiple retrieval lists, merging results for improved relevance. 68 | - **Retriever and RAG Chain Setup:** Constructs a retrieval chain for answering queries, using fused rankings and RAG chains to pull contextually relevant information. 69 | - **Cohere Re-Ranking:** Demonstrates re-ranking with Cohere’s model for additional contextual compression and refinement. 70 | - **CRAG and Self-RAG Retrieval:** Explores advanced retrieval approaches like CRAG and Self-RAG, with links to examples. 71 | - **Exploration of Long-Context Impact:** Links to resources explaining the impact of long-context retrieval on RAG models. 72 | 73 | ## Getting Started 74 | 75 | ### Pre-requisites 76 | 77 | Ensure **Python 3.11.11** (preferred) is installed on your system. Follow the platform-specific instructions below to install it if not already installed. 78 | 79 | #### macOS 80 | 1. Install [Homebrew](https://brew.sh/) if not already installed: 81 | ```bash 82 | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" 83 | ``` 84 | 2. Install Python 3.11.11: 85 | ```bash 86 | brew install python@3.11 87 | ``` 88 | 3. Verify installation: 89 | ```bash 90 | python3.11 --version 91 | ``` 92 | 93 | #### Linux 94 | 1. Update your package manager: 95 | ```bash 96 | sudo apt update 97 | ``` 98 | 2. Install Python 3.11.11: 99 | ```bash 100 | sudo apt install python3.11 python3.11-venv 101 | ``` 102 | 3. Verify installation: 103 | ```bash 104 | python3.11 --version 105 | ``` 106 | 107 | #### Windows 108 | 1. Download the Python 3.11.11 installer from [Python.org](https://www.python.org/downloads/). 109 | 2. Run the installer and ensure you check the box **"Add Python to PATH"**. 110 | 3. Verify installation: 111 | ```cmd 112 | python --version 113 | ``` 114 | --- 115 | 116 | ### Installation Instructions 117 | 118 | #### 1. Clone the Repository 119 | ```bash 120 | git clone https://github.com/bRAGAI/bRAG-langchain.git 121 | cd bRAG-langchain 122 | ``` 123 | 124 | #### 2. Create a Virtual Environment 125 | Use Python 3.11.11 to create a virtual environment: 126 | ```bash 127 | python3.11 -m venv venv 128 | ``` 129 | 130 | Activate the virtual environment: 131 | - **macOS/Linux**: 132 | ```bash 133 | source venv/bin/activate 134 | ``` 135 | - **Windows**: 136 | ```cmd 137 | venv\Scripts\activate 138 | ``` 139 | 140 | #### 3. Verify and Fix Python Version 141 | If the virtual environment defaults to a different Python version (e.g., Python 3.13): 142 | 1. Verify the current Python version inside the virtual environment: 143 | ```bash 144 | python --version 145 | ``` 146 | 2. Use Python 3.11 explicitly within the virtual environment: 147 | ```bash 148 | python3.11 149 | ``` 150 | 3. Ensure the `python` command uses Python 3.11 by creating a symbolic link: 151 | ```bash 152 | ln -sf $(which python3.11) $(dirname $(which python))/python 153 | ``` 154 | 4. Verify the fix: 155 | ```bash 156 | python --version 157 | ``` 158 | 159 | #### 4. Install Dependencies 160 | Install the required packages: 161 | ```bash 162 | pip install -r requirements.txt 163 | ``` 164 | 165 | --- 166 | 167 | ### Additional Steps 168 | 169 | #### 5. Run the Notebooks 170 | Begin with `[1]_rag_setup_overview.ipynb` to get familiar with the setup process. Proceed sequentially through the other notebooks: 171 | 172 | - `[1]_rag_setup_overview.ipynb` 173 | - `[2]_rag_with_multi_query.ipynb` 174 | - `[3]_rag_routing_and_query_construction.ipynb` 175 | - `[4]_rag_indexing_and_advanced_retrieval.ipynb` 176 | - `[5]_rag_retrieval_and_reranking.ipynb` 177 | 178 | #### 6. Set Up Environment Variables 179 | 1. Duplicate the `.env.example` file in the root directory and rename it to `.env`. 180 | 2. Add the following keys (replace with your actual values): 181 | 182 | ```env 183 | # LLM Model - Get key at https://platform.openai.com/api-keys 184 | OPENAI_API_KEY="your-api-key" 185 | 186 | # LangSmith - Get key at https://smith.langchain.com 187 | LANGCHAIN_TRACING_V2=true 188 | LANGCHAIN_ENDPOINT="https://api.smith.langchain.com" 189 | LANGCHAIN_API_KEY="your-api-key" 190 | LANGCHAIN_PROJECT="your-project-name" 191 | 192 | # Pinecone Vector Database - Get key at https://app.pinecone.io 193 | PINECONE_INDEX_NAME="your-project-index" 194 | PINECONE_API_HOST="your-host-url" 195 | PINECONE_API_KEY="your-api-key" 196 | 197 | # Cohere - Get key at https://dashboard.cohere.com/api-keys 198 | COHERE_API_KEY=your-api-key 199 | ``` 200 | 201 | --- 202 | 203 | You're now ready to use the project! 204 | 205 | ## Usage 206 | 207 | After setting up the environment and running the notebooks in sequence, you can: 208 | 209 | 1. **Experiment with Retrieval-Augmented Generation**: 210 | Use the foundational setup in `[1]_rag_setup_overview.ipynb` to understand the basics of RAG. 211 | 212 | 2. **Implement Multi-Querying**: 213 | Learn how to improve response relevance by introducing multi-querying techniques in `[2]_rag_with_multi_query.ipynb`. 214 | 215 | ## Star History 216 | 217 | 218 | 219 | 220 | 221 | Star History Chart 222 | 223 | 224 | 225 | ## Upcoming Notebooks 226 | 227 | 👨🏻‍💻 **[MistralOCR](https://mistral.ai/news/mistral-ocr) + RAG Integration** 228 | 229 | ## Contact 230 | Do you have questions or want to collaborate? Please open an issue or email Taha Ababou at taha@bragai.dev 231 | 232 | `If this project helps you, consider buying me a coffee ☕. Your support helps me keep contributing to the open-source community!` 233 |

234 | 235 | 236 | 237 |

238 | 239 |
240 | 241 | The notebooks and visual diagrams were inspired by Lance Martin's LangChain Tutorial. 242 | 243 | 244 | -------------------------------------------------------------------------------- /assets/img/pinecone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BragAI/bRAG-langchain/b7c5b3d1239e298e2a4628ce1e715e0c3db09277/assets/img/pinecone.png -------------------------------------------------------------------------------- /assets/img/rag-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BragAI/bRAG-langchain/b7c5b3d1239e298e2a4628ce1e715e0c3db09277/assets/img/rag-architecture.png -------------------------------------------------------------------------------- /docs/[1]_sources.md: -------------------------------------------------------------------------------- 1 | ### Here are all the sources used to write-up the `[1]_rag_setup_overview.ipynb` file: 2 | 3 | 1. LangSmith Documentation: https://docs.smith.langchain.com/ 4 | 2. RAG Quickstart: https://python.langchain.com/docs/tutorials/rag/ 5 | 3. Count Tokens: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb 6 | 4. Text Embedding Models: https://python.langchain.com/docs/integrations/text_embedding/openai 7 | 5. Cosine Similarity: https://platform.openai.com/docs/guides/embeddings/frequently-asked-questions 8 | 6. Document Loaders: https://python.langchain.com/docs/integrations/document_loaders/ 9 | 7. Splitter: https://python.langchain.com/docs/how_to/recursive_text_splitter/ 10 | 8. Vectorstores: https://python.langchain.com/docs/integrations/vectorstores/ 11 | 9. RAG Chains: https://python.langchain.com/docs/how_to/sequence/ 12 | 13 | These links provide additional resources and documentation related to the concepts discussed in the file. -------------------------------------------------------------------------------- /docs/[2]_sources.md: -------------------------------------------------------------------------------- 1 | ### Here are all the sources used to write-up the `[2]_rag_with_multi_query.ipynb` file: 2 | 3 | 1. LangSmith Documentation: https://docs.smith.langchain.com/ 4 | 2. Multi Query Retriever Documentation: https://python.langchain.com/docs/how_to/MultiQueryRetriever/ 5 | 3. RAG Fusion Documentation: https://github.com/langchain-ai/langchain/blob/master/cookbook/rag_fusion.ipynb?ref=blog.langchain.dev 6 | 4. Forget RAG Blog: https://medium.com/towards-data-science/forget-rag-the-future-is-rag-fusion-1147298d8ad1 7 | 5. Research Papers: 8 | 1. **Least-To-Most Prompting Enables Complex Reasoning In Large Language Models**: https://arxiv.org/pdf/2205.10625.pdf 9 | 2. **Interleaving Retrieval with Chain-of-Thought Reasoning for Knowledge-Intensive Multi-Step Questions**: https://arxiv.org/abs/2212.10509.pdf 10 | 3. **Take A Step Back: Evoking Reasoning Via Abstraction In Large Language Models**: https://arxiv.org/pdf/2310.06117.pdf 11 | 4. **HyDE Paper**: https://arxiv.org/abs/2212.10496 12 | 6. HyDE Documentation: https://github.com/langchain-ai/langchain/blob/master/cookbook/hypothetical_document_embeddings.ipynb -------------------------------------------------------------------------------- /docs/[3]_sources.md: -------------------------------------------------------------------------------- 1 | ### Here are all the sources used to write-up the `[3]_rag_routing_and_query_construction.ipynb` file: 2 | 3 | 1. https://docs.smith.langchain.com/ (LangSmith documentation) 4 | 2. https://python.langchain.com/docs/how_to/routing/ (Expression language & Embedding Router cookbook) 5 | 3. https://smith.langchain.com/public/c2ca61b4-3810-45d0-a156-3d6a73e9ee2a/r (Trace example) 6 | 4. https://smith.langchain.com/public/98c25405-2631-4de8-b12a-1891aded3359/r (Additional trace example) 7 | 5. https://blog.langchain.dev/query-construction/ (Query construction blog post) 8 | 6. https://blog.langchain.dev/enhancing-rag-based-applications-accuracy-by-constructing-and-leveraging-knowledge-graphs/ (Knowledge graphs in RAG) 9 | 7. https://python.langchain.com/v0.1/docs/use_cases/query_analysis/ (Query analysis documentation) 10 | 8. https://python.langchain.com/docs/how_to/self_query/ (Self-querying documentation) 11 | 12 | These links provide additional resources and documentation related to the concepts discussed in the file. -------------------------------------------------------------------------------- /docs/[4]_sources.md: -------------------------------------------------------------------------------- 1 | ### Here are all the sources used to write-up the `[4]_rag_indexing_and_advanced_retrieval.ipynb` file: 2 | 3 | 1. https://www.youtube.com/watch?v=8OJC21T2SL4 (Greg Kamradt's video on document chunking) 4 | 2. https://docs.smith.langchain.com/ (LangSmith documentation) 5 | 3. https://blog.langchain.dev/semi-structured-multi-modal-rag/ (Semi-structured multi-modal RAG) 6 | 4. https://python.langchain.com/docs/how_to/multi_vector/ (Multi-vector retrievers) 7 | 5. https://arxiv.org/abs/2312.06648 (Dense X Retrieval research paper) 8 | 6. https://python.langchain.com/docs/how_to/parent_document_retriever/ (Parent Document Retriever documentation) 9 | 7. https://www.youtube.com/watch?v=jbGchdTL7d0 (LangChain video on advanced retrieval) 10 | 8. https://arxiv.org/pdf/2401.18059 (RAPTOR research paper) 11 | 9. https://github.com/langchain-ai/langchain/blob/master/cookbook/RAPTOR.ipynb (RAPTOR implementation cookbook) 12 | 13 | These links provide additional resources and documentation related to the concepts discussed in the file. -------------------------------------------------------------------------------- /docs/[5]_sources.md: -------------------------------------------------------------------------------- 1 | ### Here are all the sources used to write-up the `[5]_rag_retrieval_and_reranking.ipynb` file: 2 | 3 | 1. https://docs.smith.langchain.com/ (LangSmith documentation) 4 | 2. https://python.langchain.com/docs/integrations/retrievers/cohere-reranker#doing-reranking-with-coherererank (Cohere Re-Rank) 5 | 3. https://txt.cohere.com/rerank/ (Cohere Rerank documentation) 6 | 4. https://www.youtube.com/watch?v=E2shqsYwxck (CRAG Deep Dive) 7 | 5. https://github.com/langchain-ai/langgraph/blob/main/examples/rag/langgraph_crag.ipynb (CRAG notebook) 8 | 6. https://github.com/langchain-ai/langgraph/blob/main/examples/rag/langgraph_crag.ipynb (CRAG notebook) 9 | 7. https://github.com/langchain-ai/langgraph/tree/main/examples/rag (RAG examples) 10 | 8. https://www.youtube.com/watch?v=SsHUNfhF32s (Long context impact deep dive) 11 | 9. https://docs.google.com/presentation/d/1mJUiPBdtf58NfuSEQ7pVSEQ2Oqmek7F1i4gBwR6JDss/edit#slide=id.g26c0cb8dc66_0_0 (Slides) 12 | 13 | These links provide additional resources and documentation related to the concepts discussed in the file. -------------------------------------------------------------------------------- /full_basic_rag.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# bRAG: Basic (naive) RAG Implementation\n", 8 | "\n", 9 | "This notebook demonstrates a complete implementation of a basic RAG system that enables question-answering over PDF documents. \n", 10 | "The implementation is model, database, and document loader agnostic, though it's currently configured with:\n", 11 | "- LLM: OpenAI GPT-3.5-turbo\n", 12 | "- Vector Database: Pinecone\n", 13 | "- Document Loader: PyPDFLoader\n", 14 | "\n", 15 | "The system combines several key components:\n", 16 | "1. Document Loading: Loads PDF documents (extensible to other document types)\n", 17 | "2. Text Processing: Splits documents into manageable chunks\n", 18 | "3. Vector Operations:\n", 19 | " - Embeds text using OpenAI's embedding model\n", 20 | " - Stores vectors in Pinecone vector database\n", 21 | "4. Retrieval System: Implements efficient document retrieval\n", 22 | "5. LLM Integration: Uses OpenAI's GPT model for generating responses\n", 23 | "\n", 24 | "All components can be swapped out for alternatives (e.g., different LLMs, vector stores, or document loaders) \n", 25 | "while maintaining the same overall architecture.\n", 26 | "\n", 27 | "This implementation serves as a foundation for building more complex RAG applications\n", 28 | "and can be customized based on specific use cases.\n", 29 | "\n", 30 | "----------------------------------------" 31 | ] 32 | }, 33 | { 34 | "cell_type": "markdown", 35 | "metadata": {}, 36 | "source": [ 37 | "## Pre-requisites (optional but recommended)\n", 38 | "\n", 39 | "### Only do the first step if you have never created a virtual environment for this repository. Otherwise, make sure that the Python Kernel that you selected is from your `venv/` folder." 40 | ] 41 | }, 42 | { 43 | "cell_type": "code", 44 | "execution_count": null, 45 | "metadata": {}, 46 | "outputs": [], 47 | "source": [ 48 | "# Create virtual environment\n", 49 | "! python -m venv venv" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": {}, 56 | "outputs": [], 57 | "source": [ 58 | "# Activate virtual Python environment\n", 59 | "! source venv/bin/activate" 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": null, 65 | "metadata": {}, 66 | "outputs": [ 67 | { 68 | "name": "stdout", 69 | "output_type": "stream", 70 | "text": [ 71 | "/Users/taha/Desktop/bRAGAI/code/gh/bRAG-langchain/venv/bin/python\n" 72 | ] 73 | } 74 | ], 75 | "source": [ 76 | "# If your Python is not from your venv path, ensure that your IDE's kernel selection (on the top right corner) is set to the correct path \n", 77 | "# (your path output should contain \"...venv/bin/python\")\n", 78 | "\n", 79 | "! which python" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "# Install all packages\n", 89 | "! pip install -r requirements.txt --quiet" 90 | ] 91 | }, 92 | { 93 | "cell_type": "markdown", 94 | "metadata": {}, 95 | "source": [ 96 | "## Environment\n", 97 | "\n", 98 | "`(1) Packages`" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": 2, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "import os\n", 108 | "from dotenv import load_dotenv\n", 109 | "\n", 110 | "# Load all environment variables from .env file\n", 111 | "load_dotenv()\n", 112 | "\n", 113 | "# Access the environment variables\n", 114 | "langchain_tracing_v2 = os.getenv('LANGCHAIN_TRACING_V2')\n", 115 | "langchain_endpoint = os.getenv('LANGCHAIN_ENDPOINT')\n", 116 | "langchain_api_key = os.getenv('LANGCHAIN_API_KEY')\n", 117 | "\n", 118 | "## LLM\n", 119 | "openai_api_key = os.getenv('OPENAI_API_KEY')\n", 120 | "\n", 121 | "## Pinecone Vector Database\n", 122 | "pinecone_api_key = os.getenv('PINECONE_API_KEY')\n", 123 | "pinecone_api_host = os.getenv('PINECONE_API_HOST')\n", 124 | "index_name = os.getenv('PINECONE_INDEX_NAME')\n" 125 | ] 126 | }, 127 | { 128 | "cell_type": "markdown", 129 | "metadata": {}, 130 | "source": [ 131 | "`(2) LangSmith`\n", 132 | "\n", 133 | "https://docs.smith.langchain.com/" 134 | ] 135 | }, 136 | { 137 | "cell_type": "code", 138 | "execution_count": 3, 139 | "metadata": {}, 140 | "outputs": [], 141 | "source": [ 142 | "os.environ['LANGCHAIN_TRACING_V2'] = langchain_tracing_v2\n", 143 | "os.environ['LANGCHAIN_ENDPOINT'] = langchain_endpoint\n", 144 | "os.environ['LANGCHAIN_API_KEY'] = langchain_api_key" 145 | ] 146 | }, 147 | { 148 | "cell_type": "markdown", 149 | "metadata": {}, 150 | "source": [ 151 | "`(3) API Keys`" 152 | ] 153 | }, 154 | { 155 | "cell_type": "code", 156 | "execution_count": 4, 157 | "metadata": {}, 158 | "outputs": [], 159 | "source": [ 160 | "os.environ['OPENAI_API_KEY'] = openai_api_key\n", 161 | "openai_model = \"gpt-3.5-turbo\"\n", 162 | "\n", 163 | "#Pinecone keys\n", 164 | "os.environ['PINECONE_API_KEY'] = pinecone_api_key\n", 165 | "os.environ['PINECONE_API_HOST'] = pinecone_api_host\n", 166 | "os.environ['PINECONE_INDEX_NAME'] = index_name" 167 | ] 168 | }, 169 | { 170 | "cell_type": "markdown", 171 | "metadata": {}, 172 | "source": [ 173 | "`(4) Pinecone Init`" 174 | ] 175 | }, 176 | { 177 | "cell_type": "code", 178 | "execution_count": 5, 179 | "metadata": {}, 180 | "outputs": [ 181 | { 182 | "name": "stderr", 183 | "output_type": "stream", 184 | "text": [ 185 | "/Users/taha/Desktop/bRAGAI/code/gh/bRAG-langchain/venv/lib/python3.11/site-packages/pinecone/data/index.py:1: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", 186 | " from tqdm.autonotebook import tqdm\n" 187 | ] 188 | } 189 | ], 190 | "source": [ 191 | "from pinecone import Pinecone\n", 192 | "\n", 193 | "pc = Pinecone(api_key=os.environ['PINECONE_API_KEY'])\n", 194 | "index = pc.Index(os.environ['PINECONE_INDEX_NAME'])" 195 | ] 196 | }, 197 | { 198 | "cell_type": "markdown", 199 | "metadata": {}, 200 | "source": [ 201 | "## Full RAG App (Basic)" 202 | ] 203 | }, 204 | { 205 | "cell_type": "code", 206 | "execution_count": 6, 207 | "metadata": {}, 208 | "outputs": [], 209 | "source": [ 210 | "from langchain_community.document_loaders import PyPDFLoader\n", 211 | "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", 212 | "from langchain.vectorstores import Pinecone\n", 213 | "from langchain_core.output_parsers import StrOutputParser\n", 214 | "from langchain_core.runnables import RunnablePassthrough\n", 215 | "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n", 216 | "from langchain.prompts import ChatPromptTemplate\n", 217 | "\n", 218 | "#### INDEXING ####\n", 219 | "\n", 220 | "pdf_file_path = \"test/langchain_turing.pdf\"\n", 221 | "loader = PyPDFLoader(pdf_file_path)\n", 222 | "\n", 223 | "docs = loader.load()\n", 224 | "\n", 225 | "# Split\n", 226 | "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n", 227 | "splits = text_splitter.split_documents(docs)\n", 228 | "\n", 229 | "# Embed\n", 230 | "vectorstore = Pinecone.from_documents(\n", 231 | " documents=splits, \n", 232 | " embedding=OpenAIEmbeddings(model=\"text-embedding-3-large\"), \n", 233 | " index_name=index_name\n", 234 | ")\n", 235 | "\n", 236 | "retriever = vectorstore.as_retriever()\n", 237 | "\n", 238 | "#### RETRIEVAL and GENERATION ####\n", 239 | "\n", 240 | "# Prompt\n", 241 | "template = \"\"\"Answer the question based only on the following context:\n", 242 | "{context}\n", 243 | "\n", 244 | "Question: {question}\n", 245 | "\"\"\"\n", 246 | "\n", 247 | "prompt = ChatPromptTemplate.from_template(template)\n", 248 | "\n", 249 | "# LLM\n", 250 | "llm = ChatOpenAI(model_name=openai_model, temperature=0.1)\n", 251 | "\n", 252 | "# Post-processing\n", 253 | "def format_docs(docs):\n", 254 | " return \"\\n\\n\".join(doc.page_content for doc in docs)\n", 255 | "\n", 256 | "# Chain\n", 257 | "rag_chain = (\n", 258 | " {\"context\": retriever | format_docs, \"question\": RunnablePassthrough()}\n", 259 | " | prompt\n", 260 | " | llm\n", 261 | " | StrOutputParser()\n", 262 | ")" 263 | ] 264 | }, 265 | { 266 | "cell_type": "code", 267 | "execution_count": 11, 268 | "metadata": {}, 269 | "outputs": [ 270 | { 271 | "name": "stdout", 272 | "output_type": "stream", 273 | "text": [ 274 | "('This document is about LangChain, a modular framework for Large Language '\n", 275 | " 'Models (LLMs), focusing on its architecture, core components, capabilities, '\n", 276 | " 'challenges, security implications, and applications in NLP. It provides '\n", 277 | " 'insights into how LangChain can be leveraged to build innovative and secure '\n", 278 | " 'LLM-powered applications tailored to specific needs.')\n" 279 | ] 280 | } 281 | ], 282 | "source": [ 283 | "# Question\n", 284 | "from pprint import pprint\n", 285 | "\n", 286 | "pprint(rag_chain.invoke(\"What is this document about?\"))" 287 | ] 288 | } 289 | ], 290 | "metadata": { 291 | "kernelspec": { 292 | "display_name": "venv", 293 | "language": "python", 294 | "name": "python3" 295 | }, 296 | "language_info": { 297 | "codemirror_mode": { 298 | "name": "ipython", 299 | "version": 3 300 | }, 301 | "file_extension": ".py", 302 | "mimetype": "text/x-python", 303 | "name": "python", 304 | "nbconvert_exporter": "python", 305 | "pygments_lexer": "ipython3", 306 | "version": "3.11.11" 307 | } 308 | }, 309 | "nbformat": 4, 310 | "nbformat_minor": 2 311 | } 312 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ipykernel 2 | ragatouille 3 | python-dotenv 4 | langchain 5 | langchain-community 6 | langchain-core 7 | langchain-openai 8 | pinecone-client 9 | beautifulsoup4 10 | tiktoken 11 | numpy 12 | langchainhub 13 | chromadb 14 | youtube-transcript-api 15 | pytube 16 | cohere 17 | pypdf 18 | yt_dlp -------------------------------------------------------------------------------- /test/langchain_turing.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BragAI/bRAG-langchain/b7c5b3d1239e298e2a4628ce1e715e0c3db09277/test/langchain_turing.pdf --------------------------------------------------------------------------------