├── .gitignore ├── CODE_OF_CONDUCT.md ├── Journey 1 - RAG and Knowledge Retrieval Fundamentals ├── README.md └── sample │ ├── .env.sample │ ├── 1-RAG-Fundamentals.ipynb │ └── data │ ├── Benefit_Options.pdf │ ├── Northwind_Health_Plus_Benefits_Details.pdf │ ├── Northwind_Standard_Benefits_Details.pdf │ ├── PerksPlus.pdf │ ├── employee_handbook.pdf │ └── role_library.pdf ├── Journey 2 - Build the Ultimate Retrieval System for RAG ├── README.md └── sample │ ├── .env.sample │ ├── 2-Build-The-Ultimate-Retrieval-System-For-RAG.ipynb │ ├── MSFT_cloud_architecture_contoso.pdf │ └── sample.env ├── Journey 3 - Optimize your Vector Index for Scale ├── README.md └── sample │ ├── .env.sample │ └── 3-Vector-Compression.ipynb ├── Journey 4 - RAG for All your Data Multimodal and Beyond ├── README.md └── sample │ ├── README.md │ ├── basic-vector-workflow │ ├── azure-search-vector-python-sample-requirements.txt │ └── azure-search-vector-python-sample.ipynb │ └── integrated-vectorization │ ├── azure-search-integrated-vectorization-sample-requirements.txt │ └── azure-search-integrated-vectorization-sample.ipynb ├── Journey 5 - Hero use cases for RAG ├── README.md └── sample │ └── README.md ├── Journey Bonus - Agentic RAG ├── .gitignore ├── README.md └── sample │ ├── .chainlit │ └── config.toml │ ├── README.md │ ├── agents.py │ ├── app.py │ ├── chainlit.md │ ├── data │ ├── Benefit_Options.pdf │ ├── Northwind_Health_Plus_Benefits_Details.pdf │ └── Northwind_Standard_Benefits_Details.pdf │ ├── examples │ ├── Reflection Example 1.json │ ├── Reflection Example 2.json │ ├── Reflection Example 3.json │ └── Web Search Reflection.json │ ├── ingest.py │ ├── media │ └── demo.gif │ ├── models.py │ ├── prompts │ ├── chat_answer_question.prompty │ ├── chat_query_rewrite.prompty │ └── chat_reflect_answer.prompty │ ├── public │ ├── elements │ │ ├── EvaluationResults.jsx │ │ └── ReflectionAction.jsx │ ├── favicon.ico │ ├── logo_dark.png │ └── logo_light.png │ ├── requirements.txt │ ├── sample-index │ ├── data.json │ └── schema.json │ └── sample.env ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md └── images ├── agenda-content.png ├── banner-rag-time.png ├── j1.png ├── j2.png ├── j3.png ├── j4.png ├── j5.png ├── journeys-weekly.png ├── repo-banner.png ├── speakers.png └── visuals ├── J1-recap.png ├── J2-recap.png ├── J3-recap.png ├── J4-recap.png └── J5-recap.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | cover/ 52 | 53 | # Jupyter Notebook 54 | .ipynb_checkpoints 55 | # Jupyter working environment setting file 56 | .jupyter 57 | 58 | # pyenv 59 | .python-version 60 | 61 | # Environments 62 | .env 63 | .venv 64 | env/ 65 | venv/ 66 | ENV/ 67 | env.bak/ 68 | venv.bak/ 69 | 70 | # Spyder project settings 71 | .spyderproject 72 | .spyproject 73 | 74 | # Rope project settings 75 | .ropeproject 76 | 77 | # mkdocs documentation 78 | /site 79 | 80 | # mypy 81 | .mypy_cache/ 82 | .dmypy.json 83 | dmypy.json 84 | 85 | # Pyre type checker 86 | .pyre/ 87 | 88 | # pytype static type analyzer 89 | .pytype/ 90 | 91 | # profiling data 92 | .prof 93 | 94 | # vscode 95 | .vscode/ 96 | 97 | # Editors and IDEs 98 | .idea/ 99 | .project 100 | .classpath 101 | .cproject 102 | .settings/ 103 | *.swp 104 | *.swo 105 | 106 | # User-specific stuff 107 | *.sublime-workspace 108 | 109 | # Operating System Files 110 | .DS_Store 111 | .DS_Store? 112 | ._* 113 | .Spotlight-V100 114 | .Trashes 115 | ehthumbs.db 116 | Thumbs.db 117 | 118 | # Ignore SQLite database files 119 | *.sqlite 120 | *.sqlite3 121 | 122 | # Ignore Streamlit credentials 123 | .streamlit/credentials.toml 124 | 125 | # Optionally ignore local Streamlit config if it's environment-specific 126 | .streamlit/config.toml 127 | *.pptx 128 | .chainlit -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /Journey 1 - RAG and Knowledge Retrieval Fundamentals/README.md: -------------------------------------------------------------------------------- 1 | # Journey 1: RAG and Knowledge Retrieval Fundamentals 2 | 3 | [![Journey 1 video](https://github.com/user-attachments/assets/90942e69-78b5-4e47-a6b9-adc1c52b5a8d)](https://aka.ms/rag-time/journey1) 4 | 5 | ## Overview 6 | 7 | Welcome to **Journey 1: RAG and Knowledge Retrieval Fundamentals**. This space is designed to help you understand how retrieval and indexing work together to enhance AI-generated responses. 8 | 9 | In this journey, we explore how indexing makes retrieval possible, why it’s essential for scalable, real-time AI interactions, and how Azure AI Search enables intelligent data access. You'll learn how search indexing, vector embeddings help AI retrieve information before generating responses. 10 | 11 | * **📺 [Watch the session](https://aka.ms/rag-time/journey1)** 12 | * **📝 [Read the blog post](https://aka.ms/rag-time/journey1-blog)** 13 | * **🚀 [Test the sample](./sample/)** 14 | 15 | ## 🎥 Session Summary 16 | 17 | ### 🎥 AI & RAG: The Big Picture 18 | 19 | AI models are powerful, but without real-time knowledge retrieval, their responses can be outdated or generic. RAG bridges this gap by dynamically pulling relevant information before generating text, ensuring that AI remains accurate and adaptable. But for retrieval to be efficient, data must be properly indexed. A well-structured index enables AI to find and apply knowledge instantly, reducing search time and improving response quality. 20 | 21 | ### 📚 Making AI Smarter: The Role of Indexing 22 | 23 | Indexing is the backbone of AI retrieval. Without it, AI systems would struggle to locate relevant data quickly. Keyword indexing provides structured search, while vector indexing enables AI to understand meaning beyond exact word matches. Hybrid approaches combine both for maximum accuracy. Azure AI Search helps developers create scalable search solutions that integrate these methods, making AI-powered applications faster and more reliable. 24 | 25 | ### 🖍 Doodle Summary 26 | 27 | A visual summary of key takeaways is available to reinforce learning. 28 | 29 | ![Doodle summary journey 1](./../images/visuals/J1-recap.png) 30 | 31 | ## 📂 Sample Code 32 | 33 | To get hands-on experience, explore the sample implementation in the 📂 [Journey 1 Sample](./sample/) folder. 34 | 35 | ## 🔗 Additional Learning Resources 36 | - 📚 Azure AI Search Documentation: [Learn more](https://learn.microsoft.com/en-us/azure/search/) 37 | - 📝 Read the Blog for Journey 1: [RAG and knowledge retrieval fundamentals](https://aka.ms/rag-time/journey1-blog) 38 | - 💬 Join the Discussion: Ask your questions on our [Discord channel](https://aka.ms/rag-time/discord). 39 | 40 | ### Next Up: Continue to [Journey 2](./../Journey%202%20-%20Build%20the%20Ultimate%20Retrieval%20System%20for%20RAG/) for the next step—Building the Ultimate Retrieval System! 41 | -------------------------------------------------------------------------------- /Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/.env.sample: -------------------------------------------------------------------------------- 1 | AZURE_OPENAI_ENDPOINT="" 2 | AZURE_OPENAI_API_KEY="" 3 | AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME="" 4 | AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME="" 5 | AZURE_SEARCH_SERVICE_ENDPOINT="" 6 | AZURE_SEARCH_INDEX_NAME="" 7 | AZURE_SEARCH_ADMIN_KEY="" -------------------------------------------------------------------------------- /Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/1-RAG-Fundamentals.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "\n", 8 | "# Retrieval-Augmented Generation (RAG) with Azure OpenAI and Azure AI Search\n", 9 | "\n", 10 | "This notebook demonstrates how to set up and use Azure OpenAI and Azure AI Search to retrieve relevant documents using vector search and generate responses using a Retrieval-Augmented Generation (RAG) approach.\n", 11 | "\n", 12 | "## Prerequisites\n", 13 | "\n", 14 | "Before running the notebook, ensure you have the following: \n", 15 | "\n", 16 | "- [Fork](https://github.com/microsoft/rag-time/fork) the repository and clone it to your local machine by following the script below:\n", 17 | "\n", 18 | " ```bash\n", 19 | " git clone https://github.com/your-org/rag-time.git\n", 20 | " cd rag-time\n", 21 | " ```\n", 22 | "\n", 23 | "- An [Azure account](https://portal.azure.com) with proper permissions to access the following services:\n", 24 | " - An **Azure OpenAI** service with an active deployment of a **chat model** and an **embedding model**.\n", 25 | " - An **Azure AI Search** service with an index that contains vectorized text data. Follow the instructions in the [Quickstart](https://learn.microsoft.com/en-us/azure/search/search-get-started-portal-import-vectors?tabs=sample-data-storage%2Cmodel-aoai%2Cconnect-data-storage) to index the documents in [data](./../../data/) folder. \n", 26 | "- Install Python 3.8 or later from [python.org](https://python.org).\n", 27 | "\n", 28 | "## Steps to Use the Notebook\n", 29 | "\n", 30 | "### 1. Install Required Libraries\n", 31 | "\n", 32 | "Run the first code cell to install the required Python libraries:" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "metadata": {}, 39 | "outputs": [], 40 | "source": [ 41 | "!pip install azure-ai-inference[opentelemetry]\n", 42 | "!pip install azure-search-documents \n", 43 | "!pip install azure-identity\n", 44 | "!pip install openai" 45 | ] 46 | }, 47 | { 48 | "cell_type": "markdown", 49 | "metadata": {}, 50 | "source": [ 51 | "### 2. Set Up Environment Variables\n", 52 | "\n", 53 | "To store credentials securely, rename `.env.sample` file to `.env` in the same directory as the notebook and update the following variables:" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": 1, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "import os\n", 63 | "from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n", 64 | "from azure.core.credentials import AzureKeyCredential\n", 65 | "from openai import AzureOpenAI\n", 66 | "from azure.search.documents import SearchClient\n", 67 | "import dotenv\n", 68 | "from azure.search.documents.models import VectorizedQuery, VectorizableTextQuery\n", 69 | "\n", 70 | "dotenv.load_dotenv()\n", 71 | "\n", 72 | "# Load Azure OpenAI environment variables\n", 73 | "AZURE_OPENAI_ENDPOINT = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n", 74 | "AZURE_OPENAI_API_KEY = os.getenv(\"AZURE_OPENAI_API_KEY\")\n", 75 | "AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME = os.getenv(\"AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME\")\n", 76 | "\n", 77 | "# Load Azure Search environment variables\n", 78 | "AZURE_SEARCH_ENDPOINT = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n", 79 | "AZURE_SEARCH_INDEX_NAME = os.getenv(\"AZURE_SEARCH_INDEX_NAME\")\n", 80 | "AZURE_SEARCH_ADMIN_KEY = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n", 81 | "\n", 82 | "# 🔹 Initialize Azure OpenAI Client (API Key or Managed Identity)\n", 83 | "if AZURE_OPENAI_API_KEY:\n", 84 | " openai_client = AzureOpenAI(\n", 85 | " api_key=AZURE_OPENAI_API_KEY,\n", 86 | " azure_endpoint=AZURE_OPENAI_ENDPOINT,\n", 87 | " api_version=\"2024-10-21\"\n", 88 | " )\n", 89 | "else:\n", 90 | " azure_credential = DefaultAzureCredential()\n", 91 | " token_provider = get_bearer_token_provider(azure_credential, \"https://cognitiveservices.azure.com/.default\")\n", 92 | " openai_client = AzureOpenAI(\n", 93 | " azure_ad_token_provider=token_provider,\n", 94 | " azure_endpoint=AZURE_OPENAI_ENDPOINT,\n", 95 | " api_version=\"2024-10-21\"\n", 96 | " )\n", 97 | "\n", 98 | "# 🔹 Initialize Azure AI Search Client (API Key or Managed Identity)\n", 99 | "if AZURE_SEARCH_ADMIN_KEY:\n", 100 | " search_client = SearchClient(\n", 101 | " endpoint=AZURE_SEARCH_ENDPOINT,\n", 102 | " index_name=AZURE_SEARCH_INDEX_NAME,\n", 103 | " credential=AzureKeyCredential(AZURE_SEARCH_ADMIN_KEY)\n", 104 | " )\n", 105 | "else:\n", 106 | " azure_credential = DefaultAzureCredential()\n", 107 | " search_client = SearchClient(\n", 108 | " endpoint=AZURE_SEARCH_ENDPOINT,\n", 109 | " index_name=AZURE_SEARCH_INDEX_NAME,\n", 110 | " credential=azure_credential\n", 111 | " )\n", 112 | "\n", 113 | "def get_embedding(text):\n", 114 | " return openai_client.embeddings.create(\n", 115 | " model=os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYED_MODEL_NAME\"),\n", 116 | " input=text\n", 117 | " ).data[0].embedding" 118 | ] 119 | }, 120 | { 121 | "cell_type": "markdown", 122 | "metadata": {}, 123 | "source": [ 124 | "### 4. Prepare a question\n", 125 | "\n", 126 | "Define a sample question and convert it into an embedding vector:" 127 | ] 128 | }, 129 | { 130 | "cell_type": "code", 131 | "execution_count": null, 132 | "metadata": {}, 133 | "outputs": [], 134 | "source": [ 135 | "user_question = \"What is included in my Northwind Health Plus plan that is not in standard?\"\n", 136 | "user_question_vector = get_embedding(user_question)\n", 137 | "print(user_question_vector)" 138 | ] 139 | }, 140 | { 141 | "cell_type": "markdown", 142 | "metadata": {}, 143 | "source": [ 144 | "### 5. Retrieve matching documents\n", 145 | "\n", 146 | "Perform a vector search in Azure AI Search to retrieve relevant document chunks:" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": 6, 152 | "metadata": {}, 153 | "outputs": [ 154 | { 155 | "name": "stdout", 156 | "output_type": "stream", 157 | "text": [ 158 | "Chunk ID: 10331bdf9a20_aHR0cHM6Ly9zdGZzdW5hdmFsYXNhbmRib3hkZXZldXMuYmxvYi5jb3JlLndpbmRvd3MubmV0L2NvbnRvc28taHIvQmVuZWZpdF9PcHRpb25zLnBkZg2_pages_1\n", 159 | "Title: Benefit_Options.pdf\n", 160 | "Text: a variety of in-network providers, including primary care \n", 161 | "physicians, specialists, hospitals, and pharmacies. This plan does not offer coverage for emergency \n", 162 | "services, mental health and substance abuse coverage, or out-of-network services.\n", 163 | "\n", 164 | "Comparison of Plans \n", 165 | "Both plans offer coverage for routine physicals, well-child visits, immunizations, and other preventive \n", 166 | "care services. The plans also cover preventive care services such as mammograms, colonoscopies, and \n", 167 | "other cancer screenings. \n", 168 | "\n", 169 | "Northwind Health Plus offers more comprehensive coverage than Northwind Standard. This plan offers \n", 170 | "coverage for emergency services, both in-network and out-of-network, as well as mental health and \n", 171 | "substance abuse coverage. Northwind Standard does not offer coverage for emergency services, mental \n", 172 | "health and substance abuse coverage, or out-of-network services. \n", 173 | "\n", 174 | "Both plans offer coverage for prescription drugs. Northwind Health Plus offers a wider range of \n", 175 | "prescription drug coverage than Northwind Standard. Northwind Health Plus covers generic, brand-\n", 176 | "name, and specialty drugs, while Northwind Standard only covers generic and brand-name drugs. \n", 177 | "\n", 178 | "Both plans offer coverage for vision and dental services. Northwind Health Plus offers coverage for vision \n", 179 | "exams, glasses, and contact lenses, as well as dental exams, cleanings, and fillings. Northwind Standard \n", 180 | "only offers coverage for vision exams and glasses. \n", 181 | "\n", 182 | "Both plans offer coverage for medical services. Northwind Health Plus offers coverage for hospital stays, \n", 183 | "doctor visits, lab tests, and X-rays. Northwind Standard only offers coverage for doctor visits and lab \n", 184 | "tests. \n", 185 | "\n", 186 | "Northwind Health Plus is a comprehensive plan that offers more coverage than Northwind Standard. \n", 187 | "Northwind Health Plus offers coverage for emergency services, mental health and substance abuse \n", 188 | "coverage, and out-of-network services, while Northwind Standard does not. Northwind Health Plus also\n", 189 | "\n", 190 | "Chunk ID: 10331bdf9a20_aHR0cHM6Ly9zdGZzdW5hdmFsYXNhbmRib3hkZXZldXMuYmxvYi5jb3JlLndpbmRvd3MubmV0L2NvbnRvc28taHIvQmVuZWZpdF9PcHRpb25zLnBkZg2_pages_2\n", 191 | "Title: Benefit_Options.pdf\n", 192 | "Text: for medical services. Northwind Health Plus offers coverage for hospital stays, \n", 193 | "doctor visits, lab tests, and X-rays. Northwind Standard only offers coverage for doctor visits and lab \n", 194 | "tests. \n", 195 | "\n", 196 | "Northwind Health Plus is a comprehensive plan that offers more coverage than Northwind Standard. \n", 197 | "Northwind Health Plus offers coverage for emergency services, mental health and substance abuse \n", 198 | "coverage, and out-of-network services, while Northwind Standard does not. Northwind Health Plus also \n", 199 | "\n", 200 | "\n", 201 | "\n", 202 | "offers a wider range of prescription drug coverage than Northwind Standard. Both plans offer coverage \n", 203 | "for vision and dental services, as well as medical services. \n", 204 | "\n", 205 | "Cost Comparison\n", 206 | "Contoso Electronics deducts the employee's portion of the healthcare cost from each paycheck. This \n", 207 | "means that the cost of the health insurance will be spread out over the course of the year, rather \n", 208 | "than being paid in one lump sum. The employee's portion of the cost will be calculated based on the \n", 209 | "selected health plan and the number of people covered by the insurance. The table below shows a \n", 210 | "cost comparison between the different health plans offered by Contoso Electronics:\n", 211 | "\n", 212 | "Next Steps \n", 213 | "We hope that this information has been helpful in understanding the differences between Northwind \n", 214 | "Health Plus and Northwind Standard. We are confident that you will find the right plan for you and \n", 215 | "your family. Thank you for choosing Contoso Electronics!\n", 216 | "\n", 217 | "Chunk ID: 10331bdf9a20_aHR0cHM6Ly9zdGZzdW5hdmFsYXNhbmRib3hkZXZldXMuYmxvYi5jb3JlLndpbmRvd3MubmV0L2NvbnRvc28taHIvTm9ydGh3aW5kX0hlYWx0aF9QbHVzX0JlbmVmaXRzX0RldGFpbHMucGRm0_pages_154\n", 218 | "Title: Northwind_Health_Plus_Benefits_Details.pdf\n", 219 | "Text: not a substitute for health insurance. You \n", 220 | "\n", 221 | "should still maintain health insurance coverage through an employer, a private plan, or a \n", 222 | "\n", 223 | "government-sponsored plan. The Northwind Health Plus plan is intended to supplement the \n", 224 | "\n", 225 | "coverage you have from other sources. \n", 226 | "\n", 227 | "Tips for Making the Most of Your Plan \n", 228 | "\n", 229 | "• Make sure you understand your plan documents and know what is covered and what is \n", 230 | "\n", 231 | "excluded. \n", 232 | "\n", 233 | "• Take advantage of preventive care services, such as check-ups and screenings, as these \n", 234 | "\n", 235 | "will help you stay healthy and avoid more costly treatments down the road. \n", 236 | "\n", 237 | "• Consider signing up for the Northwind Health Plus online portal, which allows you to view \n", 238 | "\n", 239 | "your health plan information, make payments, and access other resources. \n", 240 | "\n", 241 | "• Take advantage of Northwind Health’s 24/7 nurse hotline, which can provide you with \n", 242 | "\n", 243 | "medical advice and other assistance. \n", 244 | "\n", 245 | "• Use your plan’s in-network providers whenever possible, as this will help you save money. \n", 246 | "\n", 247 | "• Utilize Northwind Health’s online pharmacy service, which allows you to easily order and \n", 248 | "\n", 249 | "manage your prescriptions without leaving home. \n", 250 | "\n", 251 | "• When you receive health care services, make sure to check that the provider is in-network \n", 252 | "\n", 253 | "and that the services are covered by your plan. \n", 254 | "\n", 255 | "• Keep track of your claims and other plan information, as this will help you to better \n", 256 | "\n", 257 | "understand your plan and stay on top of your health care expenses. \n", 258 | "\n", 259 | "\n", 260 | "\n", 261 | "• Utilize Northwind Health’s customer service to answer any questions or address any \n", 262 | "\n", 263 | "concerns you may have about your plan. \n", 264 | "\n", 265 | "Healthcare Providers - Independent Contractors \n", 266 | "\n", 267 | "OTHER INFORMATION ABOUT THIS PLAN \n", 268 | "\n", 269 | "Healthcare Providers - Independent Contractors \n", 270 | "\n", 271 | "The Northwind Health Plus plan includes coverage for healthcare services provided by \n", 272 | "\n", 273 | "independent contractors. This means that services provided by independent contractors \n", 274 | "\n", 275 | "may be covered under the Northwind Health Plus plan, provided that the service is \n", 276 | "\n", 277 | "medically necessary.\n", 278 | "\n" 279 | ] 280 | } 281 | ], 282 | "source": [ 283 | "search_results = search_client.search(\n", 284 | " None,\n", 285 | " top=3,\n", 286 | " vector_queries=[\n", 287 | " VectorizableTextQuery( \n", 288 | " text=user_question, k_nearest_neighbors=3, fields=\"text_vector\"\n", 289 | " )\n", 290 | " ],\n", 291 | ")\n", 292 | "\n", 293 | "# Print Results\n", 294 | "for result in search_results:\n", 295 | " print(\"Chunk ID:\", result[\"chunk_id\"])\n", 296 | " print(\"Title:\", result[\"title\"])\n", 297 | " print(\"Text:\", result[\"chunk\"])\n", 298 | " print()" 299 | ] 300 | }, 301 | { 302 | "cell_type": "markdown", 303 | "metadata": {}, 304 | "source": [ 305 | "### 6. RAG TIME! Generate a Response\n", 306 | "\n", 307 | "Using the retrieved documents, construct a **system prompt** and generate a response with Azure OpenAI:" 308 | ] 309 | }, 310 | { 311 | "cell_type": "code", 312 | "execution_count": 5, 313 | "metadata": {}, 314 | "outputs": [ 315 | { 316 | "name": "stdout", 317 | "output_type": "stream", 318 | "text": [ 319 | "The Northwind Health Plus plan includes additional benefits such as preventive care, wellness programs, and enhanced coverage options that are not typically included in standard plans.\n" 320 | ] 321 | } 322 | ], 323 | "source": [ 324 | "# First, let's collect the context from search results\n", 325 | "context = \"\"\n", 326 | "for result in search_results:\n", 327 | " context += result[\"chunk\"] + \"\\n\\n\"\n", 328 | "\n", 329 | "SYSTEM_MESSAGE = f\"\"\"\n", 330 | "You are an AI Assistant.\n", 331 | "Be brief in your answers. Answer ONLY with the facts listed in the retrieved text.\n", 332 | "\n", 333 | "Context:\n", 334 | "{context}\n", 335 | "\"\"\"\n", 336 | "\n", 337 | "USER_MESSAGE = user_question\n", 338 | "\n", 339 | "response = openai_client.chat.completions.create(\n", 340 | " model=os.getenv(\"AZURE_OPENAI_CHAT_COMPLETION_DEPLOYED_MODEL_NAME\"),\n", 341 | " temperature=0.7,\n", 342 | " messages=[\n", 343 | " {\"role\": \"system\", \"content\": SYSTEM_MESSAGE},\n", 344 | " {\"role\": \"user\", \"content\": USER_MESSAGE},\n", 345 | " ],\n", 346 | ")\n", 347 | "\n", 348 | "answer = response.choices[0].message.content\n", 349 | "print(answer)\n" 350 | ] 351 | }, 352 | { 353 | "cell_type": "markdown", 354 | "metadata": {}, 355 | "source": [ 356 | "## Troubleshooting\n", 357 | "\n", 358 | "1. **Environment Variables Not Loaded:** Ensure you have correctly set the `.env` file or manually export them in your terminal before running the notebook.\n", 359 | "1. **Authentication Issues:** If using Managed Identity, make sure your Azure identity has proper role assignments.\n", 360 | "1. **Search Results Are Empty:** Ensure your Azure AI Search index contains vectorized data.\n", 361 | "1. **OpenAI API Errors:** Verify your deployment name and API key.\n", 362 | "\n", 363 | "## Summary\n", 364 | "\n", 365 | "This notebook demonstrates a **vector-based RAG pipeline** using Azure OpenAI and Azure AI Search. It retrieves relevant documents using vector search and generates responses using GPT-based chat completions. The approach improves the accuracy of AI responses by grounding them in real data." 366 | ] 367 | } 368 | ], 369 | "metadata": { 370 | "kernelspec": { 371 | "display_name": "Python 3", 372 | "language": "python", 373 | "name": "python3" 374 | }, 375 | "language_info": { 376 | "codemirror_mode": { 377 | "name": "ipython", 378 | "version": 3 379 | }, 380 | "file_extension": ".py", 381 | "mimetype": "text/x-python", 382 | "name": "python", 383 | "nbconvert_exporter": "python", 384 | "pygments_lexer": "ipython3", 385 | "version": "3.11.9" 386 | } 387 | }, 388 | "nbformat": 4, 389 | "nbformat_minor": 2 390 | } 391 | -------------------------------------------------------------------------------- /Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/Benefit_Options.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/Benefit_Options.pdf -------------------------------------------------------------------------------- /Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/Northwind_Health_Plus_Benefits_Details.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/Northwind_Health_Plus_Benefits_Details.pdf -------------------------------------------------------------------------------- /Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/Northwind_Standard_Benefits_Details.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/Northwind_Standard_Benefits_Details.pdf -------------------------------------------------------------------------------- /Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/PerksPlus.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/PerksPlus.pdf -------------------------------------------------------------------------------- /Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/employee_handbook.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/employee_handbook.pdf -------------------------------------------------------------------------------- /Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/role_library.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey 1 - RAG and Knowledge Retrieval Fundamentals/sample/data/role_library.pdf -------------------------------------------------------------------------------- /Journey 2 - Build the Ultimate Retrieval System for RAG/README.md: -------------------------------------------------------------------------------- 1 | # Journey 2: Building the Ultimate Retrieval System for RAG 2 | 3 | [![Journey 2 video](https://github.com/user-attachments/assets/a9e8d5da-bee8-44fb-8c34-c249d248d8e0)](https://aka.ms/rag-time/journey2) 4 | 5 | ## Overview 6 | 7 | Welcome to **Journey 2: Building the Ultimate Retrieval System for RAG**. This space is designed to help you understand how retrieval systems form the backbone of AI-powered applications. 8 | 9 | In this journey, we explore how to set up and structure retrieval using Azure AI Search, covering essential components like data ingestion, keyword and vector search, hybrid search, and semantic ranking. You’ll learn how to choose the right techniques to ensure accurate and efficient retrieval, making AI applications more effective in delivering context-aware responses. 10 | 11 | * **📺 [Watch the session](https://aka.ms/rag-time/journey2)** 12 | * **📝 [Read the blog post](https://aka.ms/rag-time/journey2-blog)** 13 | * **🚀 [Test the sample](./sample/)** 14 | 15 | ## 🎥 Session Summary 16 | 17 | ### 🎥 The Future of AI-Powered Retrieval 18 | 19 | Retrieval is the foundation of AI’s ability to provide relevant and accurate responses. As AI applications scale, the ability to efficiently access, rank, and retrieve data becomes a critical factor in their success. Azure AI Search brings together vector search, hybrid search, and semantic ranking, offering developers a powerful way to integrate retrieval directly into their AI workflows. Understanding how these technologies work together enables more reliable, cost-effective, and high-performance search experiences. 20 | 21 | ### 📚 Designing the Ultimate Retrieval System 22 | 23 | Building a retrieval system involves more than just storing data—it requires structuring, processing, and optimizing search techniques to ensure high relevance and efficiency. Data ingestion is the first step, transforming raw content into an indexed format that allows for rapid lookups. From there, retrieval strategies like keyword search, vector embeddings, and hybrid models determine how well the system understands and ranks information. Query optimization techniques, including semantic ranking and rewriting, further refine search accuracy, ensuring AI can return the most useful results to users. 24 | 25 | ### 🖍 Doodle Summary 26 | 27 | A visual summary of key takeaways is available to reinforce learning. 28 | 29 | ![Doodle summary journey 2](./../images/visuals/J2-recap.png) 30 | 31 | ## 📂 Sample Code 32 | 33 | To get hands-on experience, explore the sample implementation in the 📂 [Journey 2 Sample](./sample/) folder. 34 | 35 | ## 🔗 Additional Learning Resources 36 | 37 | - 📚 Azure AI Search Documentation: [Learn more](https://learn.microsoft.com/en-us/azure/search/) 38 | - 📝 Read the Blog for Journey 2: [Build the Ultimate Retrieval System for RAG](https://aka.ms/rag-time/journey2-blog) 39 | - 💬 Join the Discussion: Ask your questions on our [Discord channel](https://aka.ms/rag-time/discord) 40 | 41 | ### 🚀 Next Up: Continue to [Journey 3](./../Journey%203%20-%20Optimize%20your%20Vector%20Index%20for%20Scale) for the next step—Optimizing Your Vector Index for Scale! 42 | -------------------------------------------------------------------------------- /Journey 2 - Build the Ultimate Retrieval System for RAG/sample/.env.sample: -------------------------------------------------------------------------------- 1 | AZURE_SEARCH_SERVICE_ENDPOINT="" 2 | AZURE_SEARCH_INDEX="" 3 | AZURE_SEARCH_ADMIN_KEY="" # Leave blank if using Managed Identity -------------------------------------------------------------------------------- /Journey 2 - Build the Ultimate Retrieval System for RAG/sample/2-Build-The-Ultimate-Retrieval-System-For-RAG.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Advanced Search Techniques with Azure AI Search: Keyword, Vector, and Hybrid Methods\n", 8 | "\n", 9 | "This notebook demonstrates how to perform different types of searches using Azure AI Search, including keyword search, vector search, hybrid search, semantic ranking, and query rewriting.\n", 10 | "\n", 11 | "## Prerequisites\n", 12 | "\n", 13 | "Before running the notebook, ensure you have the following: \n", 14 | "\n", 15 | "- [Fork](https://github.com/microsoft/rag-time/fork) the repository and clone it to your local machine by following the script below:\n", 16 | "\n", 17 | " ```bash\n", 18 | " git clone https://github.com/your-org/rag-time.git\n", 19 | " cd rag-time\n", 20 | " ```\n", 21 | "\n", 22 | "- An [Azure account](https://portal.azure.com) with proper permissions to access the following services:\n", 23 | " - An **Azure AI Search** service with an index that contains vectorized text data. Follow the instructions in the [Quickstart](https://learn.microsoft.com/en-us/azure/search/search-get-started-portal-import-vectors?tabs=sample-data-storage%2Cmodel-aoai%2Cconnect-data-storage) to index the [MSFT_cloud_architecture_contoso.pdf](MSFT_cloud_architecture_contoso.pdf) file. \n", 24 | "- Install Python 3.8 or later from [python.org](https://python.org).\n", 25 | "\n", 26 | "## Steps to Use the Notebook\n", 27 | "\n", 28 | "### 1. Install Required Libraries\n", 29 | "\n", 30 | "Run the first code cell to install the required Python libraries:" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": null, 36 | "metadata": {}, 37 | "outputs": [], 38 | "source": [ 39 | "%pip install azure-search-documents==11.6.0b9 azure-identity python-dotenv pandas jinja2 --quiet" 40 | ] 41 | }, 42 | { 43 | "cell_type": "markdown", 44 | "metadata": {}, 45 | "source": [ 46 | "### 2. Set Up Environment Variables\n", 47 | "\n", 48 | "To store credentials securely, rename `.env.sample` file to `.env` in the same directory as the notebook and update the following variables:" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": null, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "AZURE_SEARCH_SERVICE_ENDPOINT=\"\"\n", 58 | "AZURE_SEARCH_INDEX=\"\"\n", 59 | "AZURE_SEARCH_ADMIN_KEY=\"\" # Leave blank if using Managed Identity" 60 | ] 61 | }, 62 | { 63 | "cell_type": "markdown", 64 | "metadata": {}, 65 | "source": [ 66 | "After setting up, the notebook will automatically load these values using dotenv.\n", 67 | "\n", 68 | "### 3. Load Environment Variables\n", 69 | "\n", 70 | "Run the following command to load environment variables from the .env file:" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": null, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "import os\n", 80 | "from azure.core.credentials import AzureKeyCredential\n", 81 | "from azure.identity import DefaultAzureCredential\n", 82 | "from dotenv import load_dotenv\n", 83 | "\n", 84 | "load_dotenv(override=True) # take environment variables from .env.\n", 85 | "\n", 86 | "endpoint = os.environ[\"AZURE_SEARCH_SERVICE_ENDPOINT\"]\n", 87 | "index_name = os.environ[\"AZURE_SEARCH_INDEX\"]\n", 88 | "credential = AzureKeyCredential(os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")) if os.getenv(\"AZURE_SEARCH_ADMIN_KEY\") else DefaultAzureCredential()" 89 | ] 90 | }, 91 | { 92 | "cell_type": "markdown", 93 | "metadata": {}, 94 | "source": [ 95 | "This will ensure all necessary credentials are available before setting up the API client." 96 | ] 97 | }, 98 | { 99 | "cell_type": "markdown", 100 | "metadata": {}, 101 | "source": [ 102 | "### 4. Set Up API Client and Define the Display Function\n", 103 | "\n", 104 | "Initialize the Azure AI Search Client for interacting with the Azure Search service and make the search results easier to read by defining a function that formats and displays results:" 105 | ] 106 | }, 107 | { 108 | "cell_type": "code", 109 | "execution_count": 16, 110 | "metadata": {}, 111 | "outputs": [], 112 | "source": [ 113 | "from azure.search.documents import SearchClient\n", 114 | "import pandas as pd\n", 115 | "\n", 116 | "search_client = SearchClient(endpoint, index_name, credential)\n", 117 | "\n", 118 | "def display_results(results):\n", 119 | " df = pd.json_normalize(list(results)).dropna(axis=1, how='all')\n", 120 | " df[\"chunk\"] = df[\"chunk\"].apply(lambda c: c[:300] + '...' if len(c) > 300 else c)\n", 121 | " first_cols = ['title', 'chunk', '@search.score']\n", 122 | " df = df[first_cols + [col for col in df.columns if col not in first_cols]]\n", 123 | "\n", 124 | " df = df.style.set_properties(**{\n", 125 | " 'max-width': '500px',\n", 126 | " 'text-align': 'left',\n", 127 | " 'white-space': 'normal',\n", 128 | " 'word-wrap': 'break-word'\n", 129 | " }).hide(axis=\"index\")\n", 130 | "\n", 131 | "\n", 132 | " return df\n" 133 | ] 134 | }, 135 | { 136 | "cell_type": "markdown", 137 | "metadata": {}, 138 | "source": [ 139 | "### 5. Perform Different Search Methods\n", 140 | "\n", 141 | "#### Keyword Search\n", 142 | "\n", 143 | "Execute a traditional keyword-based search:" 144 | ] 145 | }, 146 | { 147 | "cell_type": "code", 148 | "execution_count": null, 149 | "metadata": {}, 150 | "outputs": [], 151 | "source": [ 152 | "results = search_client.search(search_text=\"What is Contoso\", top=5, select=[\"title\", \"chunk\"])\n", 153 | "\n", 154 | "display_results(results)\n" 155 | ] 156 | }, 157 | { 158 | "cell_type": "markdown", 159 | "metadata": {}, 160 | "source": [ 161 | "#### Vector Search\n", 162 | "\n", 163 | "Retrieve documents using vector similarity search:" 164 | ] 165 | }, 166 | { 167 | "cell_type": "code", 168 | "execution_count": null, 169 | "metadata": {}, 170 | "outputs": [], 171 | "source": [ 172 | "from azure.search.documents.models import VectorizableTextQuery\n", 173 | "\n", 174 | "results = search_client.search(vector_queries=[VectorizableTextQuery(text=\"What is Contoso\", k_nearest_neighbors=50, fields=\"text_vector\")], top=5, select=[\"title\", \"chunk\"])\n", 175 | "\n", 176 | "display_results(results)" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "#### Hybrid Search (Keyword + Vector Search)\n", 184 | "\n", 185 | "Combine keyword and vector searches for better accuracy:" 186 | ] 187 | }, 188 | { 189 | "cell_type": "code", 190 | "execution_count": null, 191 | "metadata": {}, 192 | "outputs": [], 193 | "source": [ 194 | "results = search_client.search(\n", 195 | " search_text=\"What is Contoso\",\n", 196 | " vector_queries=[VectorizableTextQuery(text=\"What is Contoso\", k_nearest_neighbors=50, fields=\"text_vector\")],\n", 197 | " top=5,\n", 198 | " select=[\"title\", \"chunk\"]\n", 199 | ")\n", 200 | "\n", 201 | "display_results(results)" 202 | ] 203 | }, 204 | { 205 | "cell_type": "markdown", 206 | "metadata": {}, 207 | "source": [ 208 | "#### Hybrid Search + Semantic Ranker\n", 209 | "\n", 210 | "Enhance search results using a semantic ranker:" 211 | ] 212 | }, 213 | { 214 | "cell_type": "code", 215 | "execution_count": null, 216 | "metadata": {}, 217 | "outputs": [], 218 | "source": [ 219 | "results = search_client.search(\n", 220 | " search_text=\"What is Contoso\",\n", 221 | " vector_queries=[VectorizableTextQuery(text=\"What is Contoso\", k_nearest_neighbors=50, fields=\"text_vector\")],\n", 222 | " top=5,\n", 223 | " select=[\"title\", \"chunk\"],\n", 224 | " query_type=\"semantic\",\n", 225 | " semantic_configuration_name=\"ragtime2-semantic-configuration\"\n", 226 | ")\n", 227 | "\n", 228 | "display_results(results)" 229 | ] 230 | }, 231 | { 232 | "cell_type": "markdown", 233 | "metadata": {}, 234 | "source": [ 235 | "#### Hybrid Search + Semantic Ranker + Query Rewriting\n", 236 | "\n", 237 | "Use semantic ranking and query rewriting for improved relevance:" 238 | ] 239 | }, 240 | { 241 | "cell_type": "code", 242 | "execution_count": null, 243 | "metadata": {}, 244 | "outputs": [], 245 | "source": [ 246 | "results = search_client.search(\n", 247 | " search_text=\"What is Contoso\",\n", 248 | " vector_queries=[VectorizableTextQuery(text=\"What is Contoso\", k_nearest_neighbors=50, fields=\"text_vector\")],\n", 249 | " top=5,\n", 250 | " select=[\"title\", \"chunk\"],\n", 251 | " query_type=\"semantic\",\n", 252 | " semantic_configuration_name=\"ragtime2-semantic-configuration\",\n", 253 | " query_rewrites=\"generative\",\n", 254 | " query_language=\"en\"\n", 255 | ")\n", 256 | "\n", 257 | "display_results(results)" 258 | ] 259 | }, 260 | { 261 | "cell_type": "markdown", 262 | "metadata": {}, 263 | "source": [ 264 | "## Troubleshooting\n", 265 | "\n", 266 | "- **Environment Variables Not Loaded:** Ensure you have correctly set the .env file or manually export them in your terminal before running the notebook.\n", 267 | "- **Authentication Issues:** If using Managed Identity, make sure your Azure identity has proper role assignments.\n", 268 | "- **Search Results Are Empty:** Ensure your Azure AI Search index contains vectorized data.\n", 269 | "- **Query Rewriting Issues:** Ensure your search service supports semantic configurations and generative query rewrites.\n", 270 | "\n", 271 | "## Summary\n", 272 | "\n", 273 | "This notebook demonstrates different search techniques using Azure AI Search, including keyword search, vector search, hybrid search, semantic ranking, and query rewriting. The approach enhances search accuracy by leveraging vector embeddings and semantic understanding to retrieve the most relevant documents.\n", 274 | "\n" 275 | ] 276 | } 277 | ], 278 | "metadata": { 279 | "kernelspec": { 280 | "display_name": ".venv", 281 | "language": "python", 282 | "name": "python3" 283 | }, 284 | "language_info": { 285 | "codemirror_mode": { 286 | "name": "ipython", 287 | "version": 3 288 | }, 289 | "file_extension": ".py", 290 | "mimetype": "text/x-python", 291 | "name": "python", 292 | "nbconvert_exporter": "python", 293 | "pygments_lexer": "ipython3", 294 | "version": "3.12.9" 295 | } 296 | }, 297 | "nbformat": 4, 298 | "nbformat_minor": 2 299 | } 300 | -------------------------------------------------------------------------------- /Journey 2 - Build the Ultimate Retrieval System for RAG/sample/MSFT_cloud_architecture_contoso.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey 2 - Build the Ultimate Retrieval System for RAG/sample/MSFT_cloud_architecture_contoso.pdf -------------------------------------------------------------------------------- /Journey 2 - Build the Ultimate Retrieval System for RAG/sample/sample.env: -------------------------------------------------------------------------------- 1 | AZURE_SEARCH_INDEX=myindex 2 | AZURE_SEARCH_SERVICE_ENDPOINT=https://myservice.search.windows.net -------------------------------------------------------------------------------- /Journey 3 - Optimize your Vector Index for Scale/README.md: -------------------------------------------------------------------------------- 1 | # Journey 3: Optimizing Your Vector Index for Scale 2 | 3 | [![Journey 3 video](https://github.com/user-attachments/assets/85f7e477-b018-4864-bf5f-8d540c13b3e6)](https://aka.ms/rag-time/journey3) 4 | 5 | ## Overview 6 | 7 | Welcome to **Journey 3: Optimizing Your Vector Index for Scale**. This space is designed to help you understand how vector search optimization enhances efficiency, cost, and performance in AI applications. 8 | 9 | In this journey, we explore techniques to reduce storage costs, improve retrieval speed, and balance quality with efficiency. You’ll learn how quantization, dimensionality reduction, oversampling, and re-scoring help developers build AI systems that scale effectively while maintaining accuracy. 10 | 11 | * **📺 [Watch the session](https://aka.ms/rag-time/journey3)** 12 | * **📝 [Read the blog post](https://aka.ms/rag-time/journey3-blog)** 13 | * **🚀 [Test the sample](./sample/)** 14 | 15 | ## 🎥 Session Summary 16 | 17 | ### 🎥 Why Optimization Matters in AI Retrieval 18 | 19 | Scaling AI applications isn’t just about adding more data—it’s about making retrieval faster, smarter, and more cost-effective. Optimizing vector indexes ensures that AI systems can store, retrieve, and process large datasets efficiently while maintaining high-quality responses. Azure AI Search provides powerful techniques like quantization, Matryoshka Representation Learning (MRL), and hybrid search to reduce storage while keeping retrieval performance high. 20 | 21 | ### 📚 Techniques for Scaling Vector Search 22 | 23 | The size and precision of vector embeddings play a major role in AI efficiency. Quantization compresses vector data by reducing precision, cutting storage requirements by up to 96x without severely impacting retrieval accuracy. Dimensionality reduction through MRL further shrinks vector size while retaining meaning. Oversampling and re-scoring help refine results, ensuring AI still delivers high-quality responses even with compressed indexes. Combining these techniques allows developers to optimize for scale without compromising accuracy. 24 | 25 | ### 🖍 Doodle Summary 26 | 27 | A visual summary of key takeaways is available to reinforce learning. 28 | 29 | ![Doodle summary journey 3](./../images/visuals/J3-recap.png) 30 | 31 | ## 📂 Sample Code 32 | 33 | To get hands-on experience, explore the sample implementation in the 📂 [Journey 3 Sample](./sample/) folder. 34 | 35 | ## 🔗 Additional Learning Resources 36 | 37 | - 📚 Azure AI Search Documentation: [Learn more](https://learn.microsoft.com/en-us/azure/search/) 38 | - 📝 Read the Blog for Journey 3: [Build the Ultimate Retrieval System for RAG](https://aka.ms/rag-time/journey3-blog) 39 | - 💬 Join the Discussion: Ask your questions on our [Discord channel](https://aka.ms/rag-time/discord) 40 | 41 | ### 🚀 Next Up: Continue to [Journey 4](./../Journey%204%20-%20RAG%20for%20All%20your%20Data%20Multimodal%20and%20Beyond/) for the next step—RAG for all your data, multimodal and beyond! 42 | -------------------------------------------------------------------------------- /Journey 3 - Optimize your Vector Index for Scale/sample/.env.sample: -------------------------------------------------------------------------------- 1 | AZURE_SEARCH_SERVICE_ENDPOINT="https://SEARCHSERVICE.search.windows.net" 2 | AZURE_SEARCH_ADMIN_KEY= 3 | AZURE_OPENAI_SERVICE_ENDPOINT="https://OPENAISERVICE.openai.azure.com/" 4 | AZURE_OPENAI_EMBED_DEPLOYMENT="text-embedding-3-large" -------------------------------------------------------------------------------- /Journey 4 - RAG for All your Data Multimodal and Beyond/README.md: -------------------------------------------------------------------------------- 1 | # Journey 4: RAG for All Your Data – Multimodal and Beyond 2 | 3 | [![Journey 4 video](https://github.com/user-attachments/assets/b6c45f1e-6bbe-4f6e-95ee-5bb949c15b95)](https://aka.ms/rag-time/journey4) 4 | 5 | ## Overview 6 | 7 | Welcome to **Journey 4: RAG for All Your Data – Multimodal and Beyond**. This space is designed to help you understand how multimodal retrieval enhances AI applications by enabling the integration of text, images, audio, and video into AI workflows. 8 | 9 | In this journey, we explore the importance of multimodal indexing, how Azure AI Foundry simplifies development, and strategies to scale multimodal RAG solutions efficiently. You’ll learn how integrated vectorization, Azure Document Intelligence, and content understanding improve retrieval accuracy and how query techniques like multi-vector search and vector weighting optimize responses. 10 | 11 | * **📺 [Watch the session](https://aka.ms/rag-time/journey4)** 12 | * **📝 [Read the blog post](https://aka.ms/rag-time/journey4-blog)** 13 | * **🚀 [Test the sample](./sample/)** 14 | 15 | ## 🎥 Session Summary 16 | 17 | ### 🎥 The Power of Multimodal RAG 18 | 19 | AI applications today must process more than just text—images, voice, and documents all play a role in delivering richer, more intuitive interactions. Multimodal workflows allow AI to match queries to multiple types of content, improving relevance and user experience. With advancements in GPT-4o real-time and Azure AI Foundry, developers can seamlessly integrate multimodal models into their applications, creating AI systems that understand and respond across various data types. 20 | 21 | ### 📚 Indexing and Retrieval for Multimodal Data 22 | 23 | Indexing multimodal data ensures AI can retrieve the right information, regardless of format. Data ingestion and preparation help structure content for efficient search. Integrated vectorization converts different data types into a unified format, making it easier to search across text, images, and voice. Azure Document Intelligence and Content Understanding further enhance retrieval by extracting insights and analyzing semantics from unstructured data. 24 | 25 | Advanced query techniques like multi-vector search, cross-field vector search, and vector weighting refine results, ensuring AI surfaces the most relevant information for each query. These strategies help developers optimize RAG workflows for accuracy, scalability, and speed. 26 | 27 | ### 🖍 Doodle Summary 28 | 29 | A visual summary of key takeaways is available to reinforce learning. 30 | 31 | ![Doodle summary journey 4](./../images/visuals/J4-recap.png) 32 | 33 | ## 📂 Sample Code 34 | 35 | To get hands-on experience, explore the sample implementation in the 📂 [Journey 4 Sample](./sample/) folder. 36 | 37 | ## 🔗 Additional Learning Resources 38 | 39 | - 📚 Azure AI Search Documentation: [Learn more](https://learn.microsoft.com/en-us/azure/search/) 40 | - 📝 Read the Blog for Journey 4: [Build the Ultimate Retrieval System for RAG](https://aka.ms/rag-time/journey4-blog) 41 | - 💬 Join the Discussion: Ask your questions on our [Discord channel](https://aka.ms/rag-time/discord) 42 | 43 | ### 🚀 Next Up: Continue to [Journey 5](./../Journey%205%20-%20Hero%20use%20cases%20for%20RAG/) for the next step—best practices and use cases for RAG! 44 | -------------------------------------------------------------------------------- /Journey 4 - RAG for All your Data Multimodal and Beyond/sample/README.md: -------------------------------------------------------------------------------- 1 | # Journey 4 - RAG for All Your Data: Multimodal and Beyond 2 | 3 | This section provides two sample projects designed to help you get started with fundamental code for integrating vectorization and query-time techniques, aiding you in your multimodal journey. 4 | 5 | 1. **Integrated Vectorization Sample**: This sample includes code to set up your end-to-end search environment for RAG (Retrieval-Augmented Generation). It guides you through the process of chunking data, calling embedding models, and automatically applying these models at query time. 6 | 7 | 2. **Basic Vector Workflow Sample**: This sample demonstrates how to configure multi-vector search and cross-field search. It also shows how to apply vector weighting to enhance retrieval relevance. 8 | 9 | Both samples are excellent starting points for leveraging multimodal capabilities and improving your search processes. 10 | -------------------------------------------------------------------------------- /Journey 4 - RAG for All your Data Multimodal and Beyond/sample/basic-vector-workflow/azure-search-vector-python-sample-requirements.txt: -------------------------------------------------------------------------------- 1 | python-dotenv 2 | azure-search-documents==11.6.0b7 3 | azure-identity 4 | openai 5 | -------------------------------------------------------------------------------- /Journey 4 - RAG for All your Data Multimodal and Beyond/sample/integrated-vectorization/azure-search-integrated-vectorization-sample-requirements.txt: -------------------------------------------------------------------------------- 1 | python-dotenv 2 | azure-search-documents==11.6.0b8 3 | azure-storage-blob 4 | azure-identity 5 | -------------------------------------------------------------------------------- /Journey 4 - RAG for All your Data Multimodal and Beyond/sample/integrated-vectorization/azure-search-integrated-vectorization-sample.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "attachments": {}, 5 | "cell_type": "markdown", 6 | "metadata": {}, 7 | "source": [ 8 | "# Azure AI Search integrated vectorization sample\n", 9 | "\n", 10 | "This Python notebook demonstrates the [integrated vectorization](https://learn.microsoft.com/azure/search/vector-search-integrated-vectorization) features of Azure AI Search that are currently in public preview. \n", 11 | "\n", 12 | "Integrated vectorization takes a dependency on indexers and skillsets, using the Text Split skill for data chunking, and the AzureOpenAIEmbedding skill and your Azure OpenAI resorce for embedding.\n", 13 | "\n", 14 | "This example uses PDFs from the `data/documents` folder for chunking, embedding, indexing, and queries.\n", 15 | "\n", 16 | "### Prerequisites\n", 17 | "\n", 18 | "+ An Azure subscription, with [access to Azure OpenAI](https://aka.ms/oai/access).\n", 19 | " \n", 20 | "+ Azure AI Search, any tier, but we recommend Basic or higher for this workload. [Enable semantic ranker](https://learn.microsoft.com/azure/search/semantic-how-to-enable-disable) if you want to run a hybrid query with semantic ranking.\n", 21 | "\n", 22 | "+ A deployment of the `text-embedding-3-large` model on Azure OpenAI.\n", 23 | "\n", 24 | "+ Azure Blob Storage. This notebook connects to your storage account and loads a container with the sample PDFs.\n" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "### Set up a Python virtual environment in Visual Studio Code\n", 32 | "\n", 33 | "1. Open the Command Palette (Ctrl+Shift+P).\n", 34 | "1. Search for **Python: Create Environment**.\n", 35 | "1. Select **Venv**.\n", 36 | "1. Select a Python interpreter. Choose 3.10 or later.\n", 37 | "\n", 38 | "It can take a minute to set up. If you run into problems, see [Python environments in VS Code](https://code.visualstudio.com/docs/python/environments)." 39 | ] 40 | }, 41 | { 42 | "cell_type": "markdown", 43 | "metadata": {}, 44 | "source": [ 45 | "### Install packages" 46 | ] 47 | }, 48 | { 49 | "cell_type": "code", 50 | "execution_count": 1, 51 | "metadata": {}, 52 | "outputs": [], 53 | "source": [ 54 | "! pip install -r azure-search-integrated-vectorization-sample-requirements.txt --quiet" 55 | ] 56 | }, 57 | { 58 | "cell_type": "markdown", 59 | "metadata": {}, 60 | "source": [ 61 | "### Load .env file (Copy .env-sample to .env and update accordingly)\n", 62 | "\n", 63 | "Optionally, you can test the following features of integrated vectorization using this notebook by setting the appropriate environment variables below:\n", 64 | "\n", 65 | "1. [OCR](https://learn.microsoft.com/en-us/azure/search/cognitive-search-skill-ocr) every page using the built-in OCR functionality. This allows you to add page numbers for every chunk that is extracted. It requires an [AI Services account](https://learn.microsoft.com/en-us/azure/search/cognitive-search-attach-cognitive-services)\n", 66 | " 1. Set `USE_OCR` to true and specify `AZURE_AI_SERVICES_KEY` if using key-based authentication, and specify `AZURE_AI_SERVICES_ENDPOINT`.\n", 67 | "1. Use the [Document Layout Skill](https://learn.microsoft.com/en-us/azure/search/cognitive-search-skill-document-intelligence-layout) to convert PDFs and other compatible documents to markdown. It requires an [AI Services account](https://learn.microsoft.com/en-us/azure/search/cognitive-search-attach-cognitive-services) and a search service in a [supported region](https://learn.microsoft.com/en-us/azure/search/cognitive-search-attach-cognitive-services)\n", 68 | " 1. Set `USE_LAYOUT` to true and specify `AZURE_AI_SERVICES_KEY` if using key-based authentication, and specify `AZURE_AI_SERVICES_ENDPOINT`.\n" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": 29, 74 | "metadata": {}, 75 | "outputs": [], 76 | "source": [ 77 | "from dotenv import load_dotenv\n", 78 | "from azure.identity import DefaultAzureCredential\n", 79 | "from azure.core.credentials import AzureKeyCredential\n", 80 | "import os\n", 81 | "\n", 82 | "load_dotenv(override=True) # take environment variables from .env.\n", 83 | "\n", 84 | "# Variables not used here do not need to be updated in your .env file\n", 85 | "endpoint = os.environ[\"AZURE_SEARCH_SERVICE_ENDPOINT\"]\n", 86 | "credential = AzureKeyCredential(os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")) if os.getenv(\"AZURE_SEARCH_ADMIN_KEY\") else DefaultAzureCredential()\n", 87 | "index_name = os.getenv(\"AZURE_SEARCH_INDEX\", \"int-vec\")\n", 88 | "blob_connection_string = os.environ[\"BLOB_CONNECTION_STRING\"]\n", 89 | "# search blob datasource connection string is optional - defaults to blob connection string\n", 90 | "# This field is only necessary if you are using MI to connect to the data source\n", 91 | "# https://learn.microsoft.com/azure/search/search-howto-indexing-azure-blob-storage#supported-credentials-and-connection-strings\n", 92 | "search_blob_connection_string = os.getenv(\"SEARCH_BLOB_DATASOURCE_CONNECTION_STRING\", blob_connection_string)\n", 93 | "blob_container_name = os.getenv(\"BLOB_CONTAINER_NAME\", \"int-vec\")\n", 94 | "azure_openai_endpoint = os.environ[\"AZURE_OPENAI_ENDPOINT\"]\n", 95 | "azure_openai_key = os.getenv(\"AZURE_OPENAI_KEY\")\n", 96 | "azure_openai_embedding_deployment = os.getenv(\"AZURE_OPENAI_EMBEDDING_DEPLOYMENT\", \"text-embedding-3-large\")\n", 97 | "azure_openai_model_name = os.getenv(\"AZURE_OPENAI_EMBEDDING_MODEL_NAME\", \"text-embedding-3-large\")\n", 98 | "azure_openai_model_dimensions = int(os.getenv(\"AZURE_OPENAI_EMBEDDING_DIMENSIONS\", 1024))\n", 99 | "# This field is only necessary if you want to use OCR to scan PDFs in the datasource or use the Document Layout skill without a key\n", 100 | "azure_ai_services_endpoint = os.getenv(\"AZURE_AI_SERVICES_ENDPOINT\", \"\")\n", 101 | "# This field is only necessary if you want to use OCR to scan PDFs in the data source or use the Document Layout skill and you want to authenticate using a key to Azure AI Services\n", 102 | "azure_ai_services_key = os.getenv(\"AZURE_AI_SERVICES_KEY\", \"\")\n", 103 | "\n", 104 | "# set USE_OCR to enable OCR to add page numbers. It cannot be combined with the document layout skill\n", 105 | "use_ocr = os.getenv(\"USE_OCR\", \"false\") == \"true\"\n", 106 | "# set USE_LAYOUT to enable Document Intelligence Layout skill for chunking by markdown. It cannot be combined with the built-in OCR\n", 107 | "use_document_layout = os.getenv(\"USE_LAYOUT\", \"false\") == \"true\"\n", 108 | "# Deepest nesting level in markdown that should be considered. See https://learn.microsoft.com/azure/search/cognitive-search-skill-document-intelligence-layout to learn more\n", 109 | "document_layout_depth = os.getenv(\"LAYOUT_MARKDOWN_HEADER_DEPTH\", \"h3\")\n", 110 | "# OCR must be used to add page numbers\n", 111 | "add_page_numbers = use_ocr\n", 112 | "\n", 113 | "if use_ocr and use_document_layout:\n", 114 | " raise Exception(\"You can only specify one of USE_OCR or USE_LAYOUT\")" 115 | ] 116 | }, 117 | { 118 | "cell_type": "markdown", 119 | "metadata": {}, 120 | "source": [ 121 | "## Connect to Blob Storage and load documents\n", 122 | "\n", 123 | "Retrieve documents from Blob Storage. You can use the sample documents in the data/documents folder. " 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "from azure.storage.blob import BlobServiceClient \n", 133 | "import glob\n", 134 | "\n", 135 | "def upload_sample_documents(\n", 136 | " blob_connection_string: str,\n", 137 | " blob_container_name: str,\n", 138 | " documents_directory: str,\n", 139 | " # Set to false if you want to use credentials included in the blob connection string\n", 140 | " # Otherwise your identity will be used as credentials\n", 141 | " use_user_identity: bool = True,\n", 142 | " ):\n", 143 | " # Connect to Blob Storage\n", 144 | " blob_service_client = BlobServiceClient.from_connection_string(logging_enable=True, conn_str=blob_connection_string, credential=DefaultAzureCredential() if use_user_identity else None)\n", 145 | " container_client = blob_service_client.get_container_client(blob_container_name)\n", 146 | " if not container_client.exists():\n", 147 | " container_client.create_container()\n", 148 | "\n", 149 | " pdf_files = glob.glob(os.path.join(documents_directory, '*.pdf'))\n", 150 | " for file in pdf_files:\n", 151 | " with open(file, \"rb\") as data:\n", 152 | " name = os.path.basename(file)\n", 153 | " if not container_client.get_blob_client(name).exists():\n", 154 | " container_client.upload_blob(name=name, data=data)\n", 155 | "\n", 156 | "def upload_documents():\n", 157 | " upload_sample_documents(\n", 158 | " blob_connection_string=blob_connection_string,\n", 159 | " blob_container_name=blob_container_name,\n", 160 | " documents_directory=os.path.join(\"..\", \"..\", \"data\", \"documents\")\n", 161 | " )\n", 162 | "\n", 163 | "def upload_documents_with_ocr():\n", 164 | " upload_sample_documents(\n", 165 | " blob_connection_string=blob_connection_string,\n", 166 | " blob_container_name=blob_container_name,\n", 167 | " documents_directory = os.path.join(\"..\", \"..\", \"data\", \"ocrdocuments\")\n", 168 | " )\n", 169 | "\n", 170 | "def upload_documents_with_layout():\n", 171 | " upload_sample_documents(\n", 172 | " blob_connection_string=blob_connection_string,\n", 173 | " blob_container_name=blob_container_name,\n", 174 | " documents_directory = os.path.join(\"..\", \"..\", \"data\", \"layoutdocuments\")\n", 175 | " )\n", 176 | "\n", 177 | "if use_ocr:\n", 178 | " upload_documents_with_ocr()\n", 179 | "elif use_document_layout:\n", 180 | " upload_documents_with_layout()\n", 181 | "else:\n", 182 | " upload_documents()\n", 183 | "\n", 184 | "print(f\"Setup sample data in {blob_container_name}\")" 185 | ] 186 | }, 187 | { 188 | "cell_type": "markdown", 189 | "metadata": {}, 190 | "source": [ 191 | "## Create a blob data source connector on Azure AI Search" 192 | ] 193 | }, 194 | { 195 | "cell_type": "code", 196 | "execution_count": null, 197 | "metadata": {}, 198 | "outputs": [], 199 | "source": [ 200 | "from azure.search.documents.indexes import SearchIndexerClient\n", 201 | "from azure.search.documents.indexes.models import (\n", 202 | " SearchIndexerDataContainer,\n", 203 | " SearchIndexerDataSourceConnection\n", 204 | ")\n", 205 | "from azure.search.documents.indexes.models import NativeBlobSoftDeleteDeletionDetectionPolicy\n", 206 | "\n", 207 | "# Create a data source \n", 208 | "indexer_client = SearchIndexerClient(endpoint, credential)\n", 209 | "container = SearchIndexerDataContainer(name=blob_container_name)\n", 210 | "data_source_connection = SearchIndexerDataSourceConnection(\n", 211 | " name=f\"{index_name}-blob\",\n", 212 | " type=\"azureblob\",\n", 213 | " connection_string=search_blob_connection_string,\n", 214 | " container=container,\n", 215 | " data_deletion_detection_policy=NativeBlobSoftDeleteDeletionDetectionPolicy()\n", 216 | ")\n", 217 | "data_source = indexer_client.create_or_update_data_source_connection(data_source_connection)\n", 218 | "\n", 219 | "print(f\"Data source '{data_source.name}' created or updated\")" 220 | ] 221 | }, 222 | { 223 | "cell_type": "markdown", 224 | "metadata": {}, 225 | "source": [ 226 | "## Create a search index\n", 227 | "\n", 228 | "Vector and nonvector content is stored in a search index." 229 | ] 230 | }, 231 | { 232 | "cell_type": "code", 233 | "execution_count": null, 234 | "metadata": {}, 235 | "outputs": [], 236 | "source": [ 237 | "from azure.search.documents.indexes import SearchIndexClient\n", 238 | "from azure.search.documents.indexes.models import (\n", 239 | " SearchField,\n", 240 | " SearchFieldDataType,\n", 241 | " VectorSearch,\n", 242 | " HnswAlgorithmConfiguration,\n", 243 | " VectorSearchProfile,\n", 244 | " AzureOpenAIVectorizer,\n", 245 | " AzureOpenAIVectorizerParameters,\n", 246 | " SemanticConfiguration,\n", 247 | " SemanticSearch,\n", 248 | " SemanticPrioritizedFields,\n", 249 | " SemanticField,\n", 250 | " SearchIndex\n", 251 | ")\n", 252 | "\n", 253 | "# Create a search index \n", 254 | "index_client = SearchIndexClient(endpoint=endpoint, credential=credential) \n", 255 | "fields = [ \n", 256 | " SearchField(name=\"parent_id\", type=SearchFieldDataType.String, sortable=True, filterable=True, facetable=True), \n", 257 | " SearchField(name=\"title\", type=SearchFieldDataType.String), \n", 258 | " SearchField(name=\"chunk_id\", type=SearchFieldDataType.String, key=True, sortable=True, filterable=True, facetable=True, analyzer_name=\"keyword\"), \n", 259 | " SearchField(name=\"chunk\", type=SearchFieldDataType.String, sortable=False, filterable=False, facetable=False), \n", 260 | " SearchField(name=\"vector\", type=SearchFieldDataType.Collection(SearchFieldDataType.Single), vector_search_dimensions=azure_openai_model_dimensions, vector_search_profile_name=\"myHnswProfile\"), \n", 261 | "]\n", 262 | "\n", 263 | "if add_page_numbers:\n", 264 | " fields.append(\n", 265 | " SearchField(name=\"page_number\", type=SearchFieldDataType.String, sortable=True, filterable=True, facetable=False)\n", 266 | " )\n", 267 | "\n", 268 | "if use_document_layout:\n", 269 | " fields.extend([\n", 270 | " SearchField(name=\"header_1\", type=SearchFieldDataType.String, sortable=False, filterable=False, facetable=False),\n", 271 | " SearchField(name=\"header_2\", type=SearchFieldDataType.String, sortable=False, filterable=False, facetable=False),\n", 272 | " SearchField(name=\"header_3\", type=SearchFieldDataType.String, sortable=False, filterable=False, facetable=False)\n", 273 | " ])\n", 274 | " \n", 275 | "# Configure the vector search configuration \n", 276 | "vector_search = VectorSearch( \n", 277 | " algorithms=[ \n", 278 | " HnswAlgorithmConfiguration(name=\"myHnsw\"),\n", 279 | " ], \n", 280 | " profiles=[ \n", 281 | " VectorSearchProfile( \n", 282 | " name=\"myHnswProfile\", \n", 283 | " algorithm_configuration_name=\"myHnsw\", \n", 284 | " vectorizer_name=\"myOpenAI\", \n", 285 | " )\n", 286 | " ], \n", 287 | " vectorizers=[ \n", 288 | " AzureOpenAIVectorizer( \n", 289 | " vectorizer_name=\"myOpenAI\", \n", 290 | " kind=\"azureOpenAI\", \n", 291 | " parameters=AzureOpenAIVectorizerParameters( \n", 292 | " resource_url=azure_openai_endpoint, \n", 293 | " deployment_name=azure_openai_embedding_deployment,\n", 294 | " model_name=azure_openai_model_name,\n", 295 | " api_key=azure_openai_key,\n", 296 | " ),\n", 297 | " ), \n", 298 | " ], \n", 299 | ") \n", 300 | " \n", 301 | "semantic_config = SemanticConfiguration( \n", 302 | " name=\"my-semantic-config\", \n", 303 | " prioritized_fields=SemanticPrioritizedFields( \n", 304 | " content_fields=[SemanticField(field_name=\"chunk\")],\n", 305 | " title_field=SemanticField(field_name=\"title\")\n", 306 | " ), \n", 307 | ")\n", 308 | " \n", 309 | "# Create the semantic search with the configuration \n", 310 | "semantic_search = SemanticSearch(configurations=[semantic_config]) \n", 311 | " \n", 312 | "# Create the search index\n", 313 | "index = SearchIndex(name=index_name, fields=fields, vector_search=vector_search, semantic_search=semantic_search) \n", 314 | "result = index_client.create_or_update_index(index) \n", 315 | "print(f\"{result.name} created\") \n" 316 | ] 317 | }, 318 | { 319 | "cell_type": "markdown", 320 | "metadata": {}, 321 | "source": [ 322 | "## Create a skillset\n", 323 | "\n", 324 | "Skills drive integrated vectorization. [Text Split](https://learn.microsoft.com/azure/search/cognitive-search-skill-textsplit) provides data chunking. [AzureOpenAIEmbedding](https://learn.microsoft.com/azure/search/cognitive-search-skill-azure-openai-embedding) handles calls to Azure OpenAI, using the connection information you provide in the environment variables. An [indexer projection](https://learn.microsoft.com/azure/search/index-projections-concept-intro) specifies secondary indexes used for chunked data." 325 | ] 326 | }, 327 | { 328 | "cell_type": "code", 329 | "execution_count": null, 330 | "metadata": {}, 331 | "outputs": [], 332 | "source": [ 333 | "from azure.search.documents.indexes.models import (\n", 334 | " SplitSkill,\n", 335 | " InputFieldMappingEntry,\n", 336 | " OutputFieldMappingEntry,\n", 337 | " AzureOpenAIEmbeddingSkill,\n", 338 | " OcrSkill,\n", 339 | " SearchIndexerIndexProjection,\n", 340 | " SearchIndexerIndexProjectionSelector,\n", 341 | " SearchIndexerIndexProjectionsParameters,\n", 342 | " IndexProjectionMode,\n", 343 | " SearchIndexerSkillset,\n", 344 | " AIServicesAccountKey,\n", 345 | " AIServicesAccountIdentity,\n", 346 | " DocumentIntelligenceLayoutSkill\n", 347 | ")\n", 348 | "\n", 349 | "# Create a skillset name \n", 350 | "skillset_name = f\"{index_name}-skillset\"\n", 351 | "\n", 352 | "def create_ocr_skillset():\n", 353 | " ocr_skill = OcrSkill(\n", 354 | " description=\"OCR skill to scan PDFs and other images with text\",\n", 355 | " context=\"/document/normalized_images/*\",\n", 356 | " line_ending=\"Space\",\n", 357 | " default_language_code=\"en\",\n", 358 | " should_detect_orientation=True,\n", 359 | " inputs=[\n", 360 | " InputFieldMappingEntry(name=\"image\", source=\"/document/normalized_images/*\")\n", 361 | " ],\n", 362 | " outputs=[\n", 363 | " OutputFieldMappingEntry(name=\"text\", target_name=\"text\"),\n", 364 | " OutputFieldMappingEntry(name=\"layoutText\", target_name=\"layoutText\")\n", 365 | " ]\n", 366 | " )\n", 367 | "\n", 368 | " split_skill = SplitSkill( \n", 369 | " description=\"Split skill to chunk documents\", \n", 370 | " text_split_mode=\"pages\", \n", 371 | " context=\"/document/normalized_images/*\", \n", 372 | " maximum_page_length=2000, \n", 373 | " page_overlap_length=500, \n", 374 | " inputs=[ \n", 375 | " InputFieldMappingEntry(name=\"text\", source=\"/document/normalized_images/*/text\"), \n", 376 | " ], \n", 377 | " outputs=[ \n", 378 | " OutputFieldMappingEntry(name=\"textItems\", target_name=\"pages\") \n", 379 | " ]\n", 380 | " )\n", 381 | "\n", 382 | " embedding_skill = AzureOpenAIEmbeddingSkill( \n", 383 | " description=\"Skill to generate embeddings via Azure OpenAI\", \n", 384 | " context=\"/document/normalized_images/*/pages/*\", \n", 385 | " resource_url=azure_openai_endpoint, \n", 386 | " deployment_name=azure_openai_embedding_deployment, \n", 387 | " model_name=azure_openai_model_name,\n", 388 | " dimensions=azure_openai_model_dimensions,\n", 389 | " api_key=azure_openai_key, \n", 390 | " inputs=[ \n", 391 | " InputFieldMappingEntry(name=\"text\", source=\"/document/normalized_images/*/pages/*\"), \n", 392 | " ], \n", 393 | " outputs=[\n", 394 | " OutputFieldMappingEntry(name=\"embedding\", target_name=\"vector\") \n", 395 | " ]\n", 396 | " )\n", 397 | "\n", 398 | " index_projections = SearchIndexerIndexProjection( \n", 399 | " selectors=[ \n", 400 | " SearchIndexerIndexProjectionSelector( \n", 401 | " target_index_name=index_name, \n", 402 | " parent_key_field_name=\"parent_id\", \n", 403 | " source_context=\"/document/normalized_images/*/pages/*\", \n", 404 | " mappings=[\n", 405 | " InputFieldMappingEntry(name=\"chunk\", source=\"/document/normalized_images/*/pages/*\"), \n", 406 | " InputFieldMappingEntry(name=\"vector\", source=\"/document/normalized_images/*/pages/*/vector\"),\n", 407 | " InputFieldMappingEntry(name=\"title\", source=\"/document/metadata_storage_name\"),\n", 408 | " InputFieldMappingEntry(name=\"page_number\", source=\"/document/normalized_images/*/pageNumber\")\n", 409 | " ]\n", 410 | " )\n", 411 | " ], \n", 412 | " parameters=SearchIndexerIndexProjectionsParameters( \n", 413 | " projection_mode=IndexProjectionMode.SKIP_INDEXING_PARENT_DOCUMENTS \n", 414 | " ) \n", 415 | " )\n", 416 | "\n", 417 | " skills = [ocr_skill, split_skill, embedding_skill]\n", 418 | "\n", 419 | " return SearchIndexerSkillset( \n", 420 | " name=skillset_name, \n", 421 | " description=\"Skillset to chunk documents and generating embeddings\", \n", 422 | " skills=skills, \n", 423 | " index_projection=index_projections,\n", 424 | " cognitive_services_account=AIServicesAccountKey(key=azure_ai_services_key, subdomain_url=azure_ai_services_endpoint) if azure_ai_services_key else AIServicesAccountIdentity(identity=None, subdomain_url=azure_ai_services_endpoint)\n", 425 | " )\n", 426 | "\n", 427 | "def create_layout_skillset():\n", 428 | " layout_skill = DocumentIntelligenceLayoutSkill(\n", 429 | " description=\"Layout skill to read documents\",\n", 430 | " context=\"/document\",\n", 431 | " output_mode=\"oneToMany\",\n", 432 | " markdown_header_depth=\"h3\",\n", 433 | " inputs=[\n", 434 | " InputFieldMappingEntry(name=\"file_data\", source=\"/document/file_data\")\n", 435 | " ],\n", 436 | " outputs=[\n", 437 | " OutputFieldMappingEntry(name=\"markdown_document\", target_name=\"markdownDocument\")\n", 438 | " ]\n", 439 | " )\n", 440 | "\n", 441 | " split_skill = SplitSkill( \n", 442 | " description=\"Split skill to chunk documents\", \n", 443 | " text_split_mode=\"pages\", \n", 444 | " context=\"/document/markdownDocument/*\", \n", 445 | " maximum_page_length=2000, \n", 446 | " page_overlap_length=500, \n", 447 | " inputs=[ \n", 448 | " InputFieldMappingEntry(name=\"text\", source=\"/document/markdownDocument/*/content\"), \n", 449 | " ], \n", 450 | " outputs=[ \n", 451 | " OutputFieldMappingEntry(name=\"textItems\", target_name=\"pages\") \n", 452 | " ]\n", 453 | " )\n", 454 | "\n", 455 | " embedding_skill = AzureOpenAIEmbeddingSkill( \n", 456 | " description=\"Skill to generate embeddings via Azure OpenAI\", \n", 457 | " context=\"/document/markdownDocument/*/pages/*\", \n", 458 | " resource_url=azure_openai_endpoint, \n", 459 | " deployment_name=azure_openai_embedding_deployment, \n", 460 | " model_name=azure_openai_model_name,\n", 461 | " dimensions=azure_openai_model_dimensions,\n", 462 | " api_key=azure_openai_key, \n", 463 | " inputs=[ \n", 464 | " InputFieldMappingEntry(name=\"text\", source=\"/document/markdownDocument/*/pages/*\"), \n", 465 | " ], \n", 466 | " outputs=[\n", 467 | " OutputFieldMappingEntry(name=\"embedding\", target_name=\"vector\") \n", 468 | " ]\n", 469 | " )\n", 470 | "\n", 471 | " index_projections = SearchIndexerIndexProjection( \n", 472 | " selectors=[ \n", 473 | " SearchIndexerIndexProjectionSelector( \n", 474 | " target_index_name=index_name, \n", 475 | " parent_key_field_name=\"parent_id\", \n", 476 | " source_context=\"/document/markdownDocument/*/pages/*\", \n", 477 | " mappings=[\n", 478 | " InputFieldMappingEntry(name=\"chunk\", source=\"/document/markdownDocument/*/pages/*\"), \n", 479 | " InputFieldMappingEntry(name=\"vector\", source=\"/document/markdownDocument/*/pages/*/vector\"),\n", 480 | " InputFieldMappingEntry(name=\"title\", source=\"/document/metadata_storage_name\"),\n", 481 | " InputFieldMappingEntry(name=\"header_1\", source=\"/document/markdownDocument/*/sections/h1\"),\n", 482 | " InputFieldMappingEntry(name=\"header_2\", source=\"/document/markdownDocument/*/sections/h2\"),\n", 483 | " InputFieldMappingEntry(name=\"header_3\", source=\"/document/markdownDocument/*/sections/h3\"),\n", 484 | " ]\n", 485 | " )\n", 486 | " ], \n", 487 | " parameters=SearchIndexerIndexProjectionsParameters( \n", 488 | " projection_mode=IndexProjectionMode.SKIP_INDEXING_PARENT_DOCUMENTS \n", 489 | " ) \n", 490 | " )\n", 491 | "\n", 492 | " skills = [layout_skill, split_skill, embedding_skill]\n", 493 | "\n", 494 | " return SearchIndexerSkillset( \n", 495 | " name=skillset_name, \n", 496 | " description=\"Skillset to chunk documents and generating embeddings\", \n", 497 | " skills=skills, \n", 498 | " index_projection=index_projections,\n", 499 | " cognitive_services_account=AIServicesAccountKey(key=azure_ai_services_key, subdomain_url=azure_ai_services_endpoint) if azure_ai_services_key else AIServicesAccountIdentity(identity=None, subdomain_url=azure_ai_services_endpoint)\n", 500 | " )\n", 501 | "\n", 502 | "def create_skillset():\n", 503 | " split_skill = SplitSkill( \n", 504 | " description=\"Split skill to chunk documents\", \n", 505 | " text_split_mode=\"pages\", \n", 506 | " context=\"/document\", \n", 507 | " maximum_page_length=2000, \n", 508 | " page_overlap_length=500, \n", 509 | " inputs=[ \n", 510 | " InputFieldMappingEntry(name=\"text\", source=\"/document/content\"), \n", 511 | " ], \n", 512 | " outputs=[ \n", 513 | " OutputFieldMappingEntry(name=\"textItems\", target_name=\"pages\") \n", 514 | " ]\n", 515 | " )\n", 516 | "\n", 517 | " embedding_skill = AzureOpenAIEmbeddingSkill( \n", 518 | " description=\"Skill to generate embeddings via Azure OpenAI\", \n", 519 | " context=\"/document/pages/*\", \n", 520 | " resource_url=azure_openai_endpoint, \n", 521 | " deployment_name=azure_openai_embedding_deployment, \n", 522 | " model_name=azure_openai_model_name,\n", 523 | " dimensions=azure_openai_model_dimensions,\n", 524 | " api_key=azure_openai_key, \n", 525 | " inputs=[ \n", 526 | " InputFieldMappingEntry(name=\"text\", source=\"/document/pages/*\"), \n", 527 | " ], \n", 528 | " outputs=[\n", 529 | " OutputFieldMappingEntry(name=\"embedding\", target_name=\"vector\") \n", 530 | " ]\n", 531 | " )\n", 532 | "\n", 533 | " index_projections = SearchIndexerIndexProjection( \n", 534 | " selectors=[ \n", 535 | " SearchIndexerIndexProjectionSelector( \n", 536 | " target_index_name=index_name, \n", 537 | " parent_key_field_name=\"parent_id\", \n", 538 | " source_context=\"/document/pages/*\", \n", 539 | " mappings=[\n", 540 | " InputFieldMappingEntry(name=\"chunk\", source=\"/document/pages/*\"), \n", 541 | " InputFieldMappingEntry(name=\"vector\", source=\"/document/pages/*/vector\"),\n", 542 | " InputFieldMappingEntry(name=\"title\", source=\"/document/metadata_storage_name\")\n", 543 | " ]\n", 544 | " )\n", 545 | " ], \n", 546 | " parameters=SearchIndexerIndexProjectionsParameters( \n", 547 | " projection_mode=IndexProjectionMode.SKIP_INDEXING_PARENT_DOCUMENTS \n", 548 | " ) \n", 549 | " )\n", 550 | "\n", 551 | " skills = [split_skill, embedding_skill]\n", 552 | "\n", 553 | " return SearchIndexerSkillset( \n", 554 | " name=skillset_name, \n", 555 | " description=\"Skillset to chunk documents and generating embeddings\", \n", 556 | " skills=skills, \n", 557 | " index_projection=index_projections\n", 558 | " )\n", 559 | "\n", 560 | "skillset = create_ocr_skillset() if use_ocr else create_layout_skillset() if use_document_layout else create_skillset()\n", 561 | " \n", 562 | "client = SearchIndexerClient(endpoint, credential) \n", 563 | "client.create_or_update_skillset(skillset) \n", 564 | "print(f\"{skillset.name} created\") \n" 565 | ] 566 | }, 567 | { 568 | "cell_type": "markdown", 569 | "metadata": {}, 570 | "source": [ 571 | "## Create an indexer" 572 | ] 573 | }, 574 | { 575 | "cell_type": "code", 576 | "execution_count": null, 577 | "metadata": {}, 578 | "outputs": [], 579 | "source": [ 580 | "from azure.search.documents.indexes.models import (\n", 581 | " SearchIndexer,\n", 582 | " IndexingParameters,\n", 583 | " IndexingParametersConfiguration,\n", 584 | " BlobIndexerImageAction\n", 585 | ")\n", 586 | "\n", 587 | "# Create an indexer \n", 588 | "indexer_name = f\"{index_name}-indexer\" \n", 589 | "\n", 590 | "indexer_parameters = None\n", 591 | "if use_ocr:\n", 592 | " indexer_parameters = IndexingParameters(\n", 593 | " configuration=IndexingParametersConfiguration(\n", 594 | " image_action=BlobIndexerImageAction.GENERATE_NORMALIZED_IMAGE_PER_PAGE,\n", 595 | " query_timeout=None))\n", 596 | "elif use_document_layout:\n", 597 | " indexer_parameters = IndexingParameters(\n", 598 | " configuration=IndexingParametersConfiguration(\n", 599 | " allow_skillset_to_read_file_data=True,\n", 600 | " query_timeout=None))\n", 601 | "\n", 602 | "indexer = SearchIndexer( \n", 603 | " name=indexer_name, \n", 604 | " description=\"Indexer to index documents and generate embeddings\", \n", 605 | " skillset_name=skillset_name, \n", 606 | " target_index_name=index_name, \n", 607 | " data_source_name=data_source.name,\n", 608 | " parameters=indexer_parameters\n", 609 | ") \n", 610 | "\n", 611 | "indexer_client = SearchIndexerClient(endpoint, credential) \n", 612 | "indexer_result = indexer_client.create_or_update_indexer(indexer) \n", 613 | " \n", 614 | "# Run the indexer \n", 615 | "indexer_client.run_indexer(indexer_name) \n", 616 | "print(f' {indexer_name} is created and running. If queries return no results, please wait a bit and try again.') \n" 617 | ] 618 | }, 619 | { 620 | "attachments": {}, 621 | "cell_type": "markdown", 622 | "metadata": {}, 623 | "source": [ 624 | "## Perform a vector similarity search" 625 | ] 626 | }, 627 | { 628 | "cell_type": "markdown", 629 | "metadata": {}, 630 | "source": [ 631 | "This example shows a pure vector search using the vectorizable text query, all you need to do is pass in text and your vectorizer will handle the query vectorization.\n", 632 | "\n", 633 | "If you indexed the health plan PDF file, send queries that ask plan-related questions." 634 | ] 635 | }, 636 | { 637 | "cell_type": "code", 638 | "execution_count": null, 639 | "metadata": {}, 640 | "outputs": [], 641 | "source": [ 642 | "from azure.search.documents import SearchClient\n", 643 | "from azure.search.documents.models import VectorizableTextQuery\n", 644 | "\n", 645 | "# Pure Vector Search\n", 646 | "query = \"Which is more comprehensive, Northwind Health Plus vs Northwind Standard?\"\n", 647 | "if use_ocr:\n", 648 | " query = \"Who is the national director?\"\n", 649 | "if use_document_layout:\n", 650 | " query = \"What is contoso?\"\n", 651 | " \n", 652 | "search_client = SearchClient(endpoint, index_name, credential=credential)\n", 653 | "vector_query = VectorizableTextQuery(text=query, k_nearest_neighbors=1, fields=\"vector\", exhaustive=True)\n", 654 | "# Use the below query to pass in the raw vector query instead of the query vectorization\n", 655 | "# vector_query = RawVectorQuery(vector=generate_embeddings(query), k_nearest_neighbors=3, fields=\"vector\")\n", 656 | " \n", 657 | "results = search_client.search( \n", 658 | " search_text=None, \n", 659 | " vector_queries= [vector_query],\n", 660 | " top=1\n", 661 | ") \n", 662 | " \n", 663 | "for result in results: \n", 664 | " print(f\"parent_id: {result['parent_id']}\") \n", 665 | " print(f\"chunk_id: {result['chunk_id']}\") \n", 666 | " if add_page_numbers:\n", 667 | " print(f\"page_number: {result['page_number']}\")\n", 668 | " print(f\"Score: {result['@search.score']}\") \n", 669 | " print(f\"Content: {result['chunk']}\") \n" 670 | ] 671 | }, 672 | { 673 | "cell_type": "markdown", 674 | "metadata": {}, 675 | "source": [ 676 | "## Perform a hybrid search" 677 | ] 678 | }, 679 | { 680 | "cell_type": "code", 681 | "execution_count": null, 682 | "metadata": {}, 683 | "outputs": [], 684 | "source": [ 685 | "# Hybrid Search\n", 686 | "query = \"Which is more comprehensive, Northwind Health Plus vs Northwind Standard?\" \n", 687 | "if use_ocr:\n", 688 | " query = \"Who is the national director?\"\n", 689 | "if use_document_layout:\n", 690 | " query = \"What is contoso?\"\n", 691 | "\n", 692 | "search_client = SearchClient(endpoint, index_name, credential=credential)\n", 693 | "vector_query = VectorizableTextQuery(text=query, k_nearest_neighbors=1, fields=\"vector\", exhaustive=True)\n", 694 | " \n", 695 | "results = search_client.search( \n", 696 | " search_text=query, \n", 697 | " vector_queries= [vector_query],\n", 698 | " select=[\"parent_id\", \"chunk_id\", \"chunk\"],\n", 699 | " top=1\n", 700 | ") \n", 701 | " \n", 702 | "for result in results: \n", 703 | " print(f\"parent_id: {result['parent_id']}\") \n", 704 | " print(f\"chunk_id: {result['chunk_id']}\") \n", 705 | " print(f\"Score: {result['@search.score']}\") \n", 706 | " print(f\"Content: {result['chunk']}\") \n" 707 | ] 708 | }, 709 | { 710 | "cell_type": "markdown", 711 | "metadata": {}, 712 | "source": [ 713 | "## Perform a hybrid search + semantic reranking" 714 | ] 715 | }, 716 | { 717 | "cell_type": "code", 718 | "execution_count": null, 719 | "metadata": {}, 720 | "outputs": [], 721 | "source": [ 722 | "from azure.search.documents.models import (\n", 723 | " QueryType,\n", 724 | " QueryCaptionType,\n", 725 | " QueryAnswerType\n", 726 | ")\n", 727 | "# Semantic Hybrid Search\n", 728 | "query = \"Which is more comprehensive, Northwind Health Plus vs Northwind Standard?\"\n", 729 | "if use_ocr:\n", 730 | " query = \"Who is the national director?\"\n", 731 | "if use_document_layout:\n", 732 | " query = \"What is contoso?\"\n", 733 | "\n", 734 | "search_client = SearchClient(endpoint, index_name, credential)\n", 735 | "vector_query = VectorizableTextQuery(text=query, k_nearest_neighbors=1, fields=\"vector\", exhaustive=True)\n", 736 | "\n", 737 | "results = search_client.search( \n", 738 | " search_text=query,\n", 739 | " vector_queries=[vector_query],\n", 740 | " select=[\"parent_id\", \"chunk_id\", \"chunk\"],\n", 741 | " query_type=QueryType.SEMANTIC,\n", 742 | " semantic_configuration_name='my-semantic-config',\n", 743 | " query_caption=QueryCaptionType.EXTRACTIVE,\n", 744 | " query_answer=QueryAnswerType.EXTRACTIVE,\n", 745 | " top=1\n", 746 | ")\n", 747 | "\n", 748 | "semantic_answers = results.get_answers()\n", 749 | "if semantic_answers:\n", 750 | " for answer in semantic_answers:\n", 751 | " if answer.highlights:\n", 752 | " print(f\"Semantic Answer: {answer.highlights}\")\n", 753 | " else:\n", 754 | " print(f\"Semantic Answer: {answer.text}\")\n", 755 | " print(f\"Semantic Answer Score: {answer.score}\\n\")\n", 756 | "\n", 757 | "for result in results:\n", 758 | " print(f\"parent_id: {result['parent_id']}\") \n", 759 | " print(f\"chunk_id: {result['chunk_id']}\") \n", 760 | " print(f\"Reranker Score: {result['@search.reranker_score']}\")\n", 761 | " print(f\"Content: {result['chunk']}\") \n", 762 | "\n", 763 | " captions = result[\"@search.captions\"]\n", 764 | " if captions:\n", 765 | " caption = captions[0]\n", 766 | " if caption.highlights:\n", 767 | " print(f\"Caption: {caption.highlights}\\n\")\n", 768 | " else:\n", 769 | " print(f\"Caption: {caption.text}\\n\")\n" 770 | ] 771 | } 772 | ], 773 | "metadata": { 774 | "kernelspec": { 775 | "display_name": ".venv", 776 | "language": "python", 777 | "name": "python3" 778 | }, 779 | "language_info": { 780 | "codemirror_mode": { 781 | "name": "ipython", 782 | "version": 3 783 | }, 784 | "file_extension": ".py", 785 | "mimetype": "text/x-python", 786 | "name": "python", 787 | "nbconvert_exporter": "python", 788 | "pygments_lexer": "ipython3", 789 | "version": "3.12.8" 790 | }, 791 | "orig_nbformat": 4 792 | }, 793 | "nbformat": 4, 794 | "nbformat_minor": 2 795 | } 796 | -------------------------------------------------------------------------------- /Journey 5 - Hero use cases for RAG/README.md: -------------------------------------------------------------------------------- 1 | # Journey 5: Hero Use Cases for RAG 2 | 3 | [![Journey 5 video](https://github.com/user-attachments/assets/0466df79-f37a-45f4-a767-52097a83870d)](https://aka.ms/rag-time/journey5) 4 | 5 | ## Overview 6 | 7 | Welcome to Journey 5: Hero Use Cases for RAG. This space is designed to help you understand how RAG-powered AI systems are transforming industries, optimizing search experiences, and driving Responsible AI practices. 8 | 9 | In this journey, we explore real-world industry use cases, key lessons from deploying RAG, common pitfalls to avoid, and the importance of Responsible AI. You’ll also gain insights into optimizing retrieval strategies, enhancing performance, and managing costs when building AI-powered search applications. 10 | 11 | * **📺 [Watch the session](https://aka.ms/rag-time/journey5)** 12 | * **📝 [Read the blog post](https://aka.ms/rag-time/journey5-blog)** 13 | * **🚀 [Test the sample](./sample/)** 14 | 15 | ## 🎥 Session Summary 16 | 17 | ### 🔍 RAG in the Real World: Industry Use Cases 18 | 19 | RAG is revolutionizing industries by unifying fragmented data, providing context-rich responses, and addressing domain-specific challenges. From legal document retrieval to customer support automation, RAG-powered solutions are enabling businesses to make AI-powered decisions faster and more efficiently. 20 | 21 | ### 📚 Best Practices and Lessons from Deploying RAG 22 | 23 | Building a high-performing RAG system requires iteration and refinement. Starting small, leveraging open-source tools, and focusing on business problems rather than technology-first approaches can significantly improve outcomes. This session highlights key takeaways, including strategies to reduce hallucinations, maintain AI quality, and continuously measure system performance. 24 | 25 | ### ⚖ Responsible AI in RAG Workflows 26 | 27 | Ensuring fairness, transparency, and accountability in AI workflows is crucial. Responsible AI practices help mitigate risks like bias, hallucinations, and trust concerns. By implementing risk and safety evaluations, content filtering, and continuous monitoring, developers can build trustworthy, reliable AI systems that evolve over time. 28 | 29 | ### 🖍 Doodle Summary 30 | 31 | A visual summary of key takeaways is available to reinforce learning. 32 | 33 | ![Doodle summary journey 5](./../images/visuals/J5-recap.png) 34 | 35 | ## 📂 Sample Code 36 | 37 | To get hands-on experience, explore the sample implementation in the 📂 [Journey 5](./sample/) Sample folder. 38 | 39 | ## 🔗 Additional Learning Resources 40 | 41 | - 📚 Azure AI Search Documentation: [Learn more](https://learn.microsoft.com/en-us/azure/search/) 42 | - 📝 Read the Blog for Journey 5: [Build the Ultimate Retrieval System for RAG](https://aka.ms/rag-time/journey5-blog) 43 | - 💬 Join the Discussion: Ask your questions on our [Discord channel](https://aka.ms/rag-time/discord) 44 | 45 | ### Your RAG journey doesn't end here! Check out the [Bonus Journey - Agentic RAG](./../Journey%20Bonus%20-%20Agentic%20RAG) to build your first Agentic RAG solution! 🚀 46 | -------------------------------------------------------------------------------- /Journey 5 - Hero use cases for RAG/sample/README.md: -------------------------------------------------------------------------------- 1 | ## LlamaIndex RAG chat app with Azure OpenAI and Azure AI Search (JavaScript) 2 | 3 | It creates a ChatGPT-like, Retrieval Augmented Generation (RAG) agentic application, over your own documents, powered by Llamaindex (TypeScript). It uses Azure OpenAI Service to access GPT models and embedding, and Azure AI Search for data indexing and retrieval. 4 | 5 | Learn more about developing AI apps using [Azure AI Services](https://aka.ms/azai). 6 | 7 | > [!IMPORTANT] 8 | > This sample application is based on [llama-index-vector-search-javascript](https://github.com/Azure-Samples/llama-index-vector-search-javascript). If you encounter issues or have feedback, please open an issue [on the original repository](https://github.com/Azure-Samples/llama-index-vector-search-javascript). 9 | 10 | [![Open in GitHub Codespaces](https://img.shields.io/static/v1?style=for-the-badge&label=GitHub+Codespaces&message=Open&color=brightgreen&logo=github)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=890591076&machine=standardLinux32gb&devcontainer_path=.devcontainer%2Fdevcontainer.json&location=WestUs2) 11 | [![Open in Dev Containers](https://img.shields.io/static/v1?style=for-the-badge&label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/azure-samples/llama-index-vector-search-javascript) 12 | 13 | ## Important Security Notice 14 | 15 | This template, the application code and configuration it contains, has been built to showcase Microsoft Azure specific services and tools. We strongly advise our customers not to make this code part of their production environments without implementing or enabling additional security features. See our [productionizing guide](docs/prod.md) for tips, and consult the [Azure OpenAI Landing Zone reference architecture](https://techcommunity.microsoft.com/blog/azurearchitectureblog/azure-openai-landing-zone-reference-architecture/3882102) for more best practices. 16 | 17 | ## Table of Contents 18 | 19 | - [Important Security Notice](#important-security-notice) 20 | - [Table of Contents](#table-of-contents) 21 | - [Architecture Diagram](#architecture-diagram) 22 | - [Azure account requirements](#azure-account-requirements) 23 | - [Cost estimation](#cost-estimation) 24 | - [Getting Started](#getting-started) 25 | - [GitHub Codespaces](#github-codespaces) 26 | - [VS Code Dev Containers](#vs-code-dev-containers) 27 | - [Local environment](#local-environment) 28 | - [Deploying](#deploying) 29 | - [Deploying again](#deploying-again) 30 | - [Running the development server](#running-the-development-server) 31 | - [Using the app](#using-the-app) 32 | - [Clean up](#clean-up) 33 | - [Guidance](#guidance) 34 | 35 | ![Chat screen](https://github.com/Azure-Samples/llama-index-vector-search-javascript/raw/main/docs/images/llamaindex-search-javascript.png) 36 | 37 | The repo includes sample data so it's ready to try end to end. In this sample application we use one of Paul Graham's essays, [What I Worked On](https://paulgraham.com/worked.html), and the experience allows you to ask questions about this essay. 38 | 39 | ### Architecture Diagram 40 | 41 | ![RAG Architecture](https://github.com/Azure-Samples/llama-index-vector-search-javascript/raw/main/docs/images/llamaindex-search-javascript-diagram.png) 42 | 43 | ## Azure account requirements 44 | 45 | **IMPORTANT:** In order to deploy and run this example, you'll need: 46 | 47 | - **Azure account**. If you're new to Azure, [get an Azure account for free](https://azure.microsoft.com/free/cognitive-search/) and you'll get some free Azure credits to get started. 48 | - **Azure account permissions**: 49 | - Your Azure account must have `Microsoft.Authorization/roleAssignments/write` permissions, such as [Role Based Access Control Administrator](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#role-based-access-control-administrator-preview), [User Access Administrator](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#user-access-administrator), or [Owner](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#owner). If you don't have subscription-level permissions, you must be granted [RBAC](https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#role-based-access-control-administrator-preview) for an existing resource group and deploy to that existing group. 50 | - Your Azure account also needs `Microsoft.Resources/deployments/write` permissions on the subscription level. 51 | 52 | ### Cost estimation 53 | 54 | Pricing varies per region and usage, so it isn't possible to predict exact costs for your usage. 55 | However, you can try the [Azure pricing calculator](https://azure.com/e/a87a169b256e43c089015fda8182ca87) for the resources below. 56 | 57 | * Azure Container Apps: Consumption plan with 1 CPU core, 2.0 GB RAM. Pricing with Pay-as-You-Go. [Pricing](https://azure.microsoft.com/pricing/details/container-apps/) 58 | * Azure OpenAI: Standard tier, gpt-4o-mini and text-embedding-3-large models. Pricing per 1K tokens used. [Pricing](https://azure.microsoft.com/pricing/details/cognitive-services/openai-service/) 59 | * Azure AI Search: Standard tier, 1 replica, free level of semantic search. Pricing per hour. [Pricing](https://azure.microsoft.com/pricing/details/search/) 60 | * Azure Blob Storage: Standard tier with ZRS (Zone-redundant storage). Pricing per storage and read operations. [Pricing](https://azure.microsoft.com/pricing/details/storage/blobs/) 61 | * Azure Monitor: Pay-as-you-go tier. Costs based on data ingested. [Pricing](https://azure.microsoft.com/pricing/details/monitor/) 62 | 63 | To reduce costs, you can switch to free SKUs for various services, but those SKUs have limitations. 64 | 65 | To avoid unnecessary costs, remember to take down your app if it's no longer in use, 66 | either by deleting the resource group in the Portal or running `azd down`. 67 | 68 | ## Getting Started 69 | 70 | You have a few options for setting up this project. 71 | The easiest way to get started is GitHub Codespaces, since it will setup all the tools for you, 72 | but you can also [set it up locally](#local-environment) if desired. 73 | 74 | ### GitHub Codespaces 75 | 76 | You can run this repo virtually by using GitHub Codespaces, which will open a web-based VS Code in your browser: 77 | 78 | [![Open in GitHub Codespaces](https://img.shields.io/static/v1?style=for-the-badge&label=GitHub+Codespaces&message=Open&color=brightgreen&logo=github)](https://github.com/codespaces/new?hide_repo_select=true&ref=main&repo=599293758&machine=standardLinux32gb&devcontainer_path=.devcontainer%2Fdevcontainer.json&location=WestUs2) 79 | 80 | Once the codespace opens (this may take several minutes), open a terminal window. 81 | 82 | ### VS Code Dev Containers 83 | 84 | A related option is VS Code Dev Containers, which will open the project in your local VS Code using the [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers): 85 | 86 | 1. Start Docker Desktop (install it if not already installed) 87 | 2. Open the project: 88 | [![Open in Dev Containers](https://img.shields.io/static/v1?style=for-the-badge&label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/azure-samples/llama-index-vector-search-javascript) 89 | 90 | 3. In the VS Code window that opens, once the project files show up (this may take several minutes), open a terminal window. 91 | 92 | ### Local environment 93 | 94 | 1. Install the required tools: 95 | 96 | - [Azure Developer CLI](https://aka.ms/azure-dev/install) 97 | - [Node.js 20+](https://nodejs.org/download/) 98 | - [Git](https://git-scm.com/downloads) 99 | - [Powershell 7+ (pwsh)](https://github.com/powershell/powershell) - For Windows users only. 100 | - **Important**: Ensure you can run `pwsh.exe` from a PowerShell terminal. If this fails, you likely need to upgrade PowerShell. 101 | 102 | 2. Create a new folder and switch to it in the terminal. 103 | 3. Run this command to download the project code: 104 | 105 | ```shell 106 | azd init -t llama-index-vector-search-javascript 107 | ``` 108 | 109 | > [!NOTE] 110 | > This command will initialize a git repository, so you do not need to clone this repository. 111 | 112 | ## Deploying 113 | 114 | The steps below will provision Azure resources and deploy the application code to Azure Container Apps. 115 | 116 | Login to your Azure account: 117 | 118 | ```shell 119 | azd auth login 120 | ``` 121 | 122 | For GitHub Codespaces users, if the previous command fails, try: 123 | 124 | ```shell 125 | azd auth login --use-device-code 126 | ``` 127 | 128 | Create a new azd environment: 129 | 130 | ```shell 131 | azd env new 132 | ``` 133 | 134 | Enter a name that will be used for the resource group. 135 | This will create a new folder in the `.azure` folder, and set it as the active environment for any calls to `azd` going forward. 136 | 137 | Provision the infrastructure needed to run the application. 138 | 139 | ```shell 140 | azd provision 141 | ``` 142 | 143 | > [!IMPORTANT] 144 | > This application specifically requires some environment variables to be available during the packaging phase. This is why we need to provision the infra first before packaging and deploying the app. In most cases, simply running 'azd up' will package, provision and deploy your apps. 145 | 146 | Package and deploy the app to Azure: 147 | 148 | ```shell 149 | azd package 150 | azd deploy 151 | ``` 152 | 153 | > [!NOTE] 154 | > This will provision Azure resources and deploy this sample to those resources, including building the search index based on the files found in the ./data folder. 155 | 156 | After the application has been successfully deployed you will see a URL printed to the console. Click that URL to interact with the application in your browser. 157 | 158 | It will look like the following: 159 | 160 | !['Output from running azd up'](https://github.com/Azure-Samples/llama-index-vector-search-javascript/raw/main/docs/images/llamaindex-search-javascript-endpoint.png) 161 | 162 | > [!NOTE] 163 | > It may take 5-10 minutes after you see 'SUCCESS' for the application to be fully deployed. 164 | 165 | > [!IMPORTANT] 166 | > Beware that the resources created by this command will incur immediate costs, primarily from the AI Search resource. These resources may accrue costs even if you interrupt the command before it is fully executed. You can run `azd down` or delete the resources manually to avoid unnecessary spending. 167 | > 168 | > You will be prompted to select two locations, one for the majority of resources and one for the OpenAI resource, which is currently a short list. That location list is based on the [OpenAI model availability table](https://learn.microsoft.com/azure/cognitive-services/openai/concepts/models#model-summary-table-and-region-availability) and may become outdated as availability changes. 169 | 170 | 171 | ### Deploying again 172 | 173 | If you've only changed the Next.js app code in the [`app`](./app/) folder, then you don't need to re-provision the Azure resources. You can just run: 174 | 175 | ```shell 176 | azd deploy 177 | ``` 178 | 179 | If you've changed the infrastructure files ([`infra`](./infra/) folder or [`azure.yaml`](./azure.yaml)), then you'll need to re-provision the Azure resources. You can do that by running: 180 | 181 | ```shell 182 | azd up 183 | ``` 184 | 185 | ## Running the development server 186 | 187 | You can run a development server locally **after** having successfully run the `azd up` (or simply `azd provision`) command. If you haven't yet, follow the [deploying](#deploying) steps above. 188 | 189 | First, `azd auth login` to authenticate to your Azure account. 190 | 191 | Then, install the project dependencies: 192 | 193 | ``` 194 | npm install 195 | ``` 196 | 197 | Next, generate the embeddings of the documents in the [./data](./data) directory: 198 | 199 | ``` 200 | npm run generate 201 | ``` 202 | 203 | Third, run the development server: 204 | 205 | ``` 206 | npm run dev 207 | ``` 208 | 209 | Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. 210 | 211 | ## Using the app 212 | 213 | - In Azure: navigate to the Azure app deployed by `azd`. The URL is printed out when `azd` completes (as "Endpoint"), or you can find it in the Azure portal. 214 | - Running locally: navigate to [http://localhost:3000](http://localhost:3000) 215 | 216 | ## Clean up 217 | 218 | To clean up all the resources created by this sample: 219 | 220 | 1. Run `azd down` 221 | 2. When asked if you are sure you want to continue, enter `y` 222 | 3. When asked if you want to permanently delete the resources, enter `y` 223 | 224 | NOTE: you can also run `azd down --purge --force`. 225 | 226 | The resource group and all the resources will be deleted. 227 | 228 | ## Guidance 229 | 230 | You can find extensive documentation in the [docs](docs/README.md) folder: 231 | 232 | - Deploying: 233 | - [Troubleshooting deployment](docs/deploy_troubleshooting.md) 234 | - [Deploying with azd: deep dive and CI/CD](docs/azd.md) 235 | - [Enabling optional features](docs/features.md) 236 | - [Productionizing](docs/prod.md) 237 | -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | .env 3 | .files -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/README.md: -------------------------------------------------------------------------------- 1 | # Bonus Journey: Agentic RAG 2 | 3 | Welcome to the **Bonus Journey: Agentic RAG!** This journey is designed to explore the cutting-edge concept of combining Retrieval-Augmented Generation (RAG) with agentic AI—creating autonomous systems that actively seek out and incorporate new knowledge. 4 | 5 | Unlike previous journeys, this one is a bit different. We’re not hosting a dedicated session, but rather collecting all the most relevant and insightful resources to get you started with Agentic RAG. Dive into the materials, experiment with the concepts, and let us know if you’d be interested in a dedicated Office Hour to discuss it further! 6 | 7 | ## 📚 What You'll Discover 8 | 9 | How to compare 3 different RAG strategies: Basic, Single-Step Reflection, and Multi-Step Reflection 10 | 11 | ## 📂 Sample Code 12 | 13 | To get hands-on experience, explore the sample implementation in the 📂 [Bonus Journey](./sample/) Sample folder. 14 | 15 | ## 🔗 Additional Learning Resources 16 | 17 | - 📝 Read the Blog for Bonus Journey: [Agentic RAG](https://aka.ms/rag-time/journey6-blog) 18 | - 📚 Azure AI Search Documentation: [Learn more](https://learn.microsoft.com/en-us/azure/search/) 19 | 20 | ## 💡 Join the Conversation 21 | 22 | Since this is an experimental and innovative journey, we’d love to hear from you! Are you curious about building autonomous AI systems? Want to see a live office hour on this topic? Let us know through our feedback form or drop a message on [our Discord channel](https://aka.ms/rag-time/discord). 23 | -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/.chainlit/config.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | # Whether to enable telemetry (default: true). No personal data is collected. 3 | enable_telemetry = false 4 | 5 | 6 | # List of environment variables to be provided by each user to use the app. 7 | user_env = [] 8 | 9 | # Duration (in seconds) during which the session is saved when the connection is lost 10 | session_timeout = 3600 11 | 12 | # Duration (in seconds) of the user session expiry 13 | user_session_timeout = 1296000 # 15 days 14 | 15 | # Enable third parties caching (e.g., LangChain cache) 16 | cache = false 17 | 18 | # Authorized origins 19 | allow_origins = ["*"] 20 | 21 | [features] 22 | # Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript) 23 | unsafe_allow_html = false 24 | 25 | # Process and display mathematical expressions. This can clash with "$" characters in messages. 26 | latex = false 27 | 28 | # Autoscroll new user messages at the top of the window 29 | user_message_autoscroll = true 30 | 31 | # Automatically tag threads with the current chat profile (if a chat profile is used) 32 | auto_tag_thread = true 33 | 34 | # Allow users to edit their own messages 35 | edit_message = false 36 | 37 | # Authorize users to spontaneously upload files with messages 38 | [features.spontaneous_file_upload] 39 | enabled = false 40 | # Define accepted file types using MIME types 41 | # Examples: 42 | # 1. For specific file types: 43 | # accept = ["image/jpeg", "image/png", "application/pdf"] 44 | # 2. For all files of certain type: 45 | # accept = ["image/*", "audio/*", "video/*"] 46 | # 3. For specific file extensions: 47 | # accept = { "application/octet-stream" = [".xyz", ".pdb"] } 48 | # Note: Using "*/*" is not recommended as it may cause browser warnings 49 | accept = ["*/*"] 50 | max_files = 20 51 | max_size_mb = 500 52 | 53 | [features.audio] 54 | # Sample rate of the audio 55 | sample_rate = 24000 56 | 57 | [features.mcp.sse] 58 | enabled = true 59 | 60 | [features.mcp.stdio] 61 | enabled = true 62 | # Only the executables in the allow list can be used for MCP stdio server. 63 | # Only need the base name of the executable, e.g. "npx", not "/usr/bin/npx". 64 | # Please don't comment this line for now, we need it to parse the executable name. 65 | allowed_executables = [ "npx", "uvx" ] 66 | 67 | [UI] 68 | # Name of the assistant. 69 | name = "Agentic RAG Demo" 70 | 71 | # default_theme = "dark" 72 | 73 | # layout = "wide" 74 | 75 | # default_sidebar_state = "open" 76 | 77 | # Description of the assistant. This is used for HTML tags. 78 | # description = "" 79 | 80 | # Chain of Thought (CoT) display mode. Can be "hidden", "tool_call" or "full". 81 | cot = "full" 82 | 83 | # Specify a CSS file that can be used to customize the user interface. 84 | # The CSS file can be served from the public directory or via an external link. 85 | # custom_css = "/public/test.css" 86 | 87 | # Specify a JavaScript file that can be used to customize the user interface. 88 | # The JavaScript file can be served from the public directory. 89 | # custom_js = "/public/test.js" 90 | 91 | # Custom login page image, relative to public directory or external URL 92 | # login_page_image = "/public/custom-background.jpg" 93 | 94 | # Custom login page image filter (Tailwind internal filters, no dark/light variants) 95 | # login_page_image_filter = "brightness-50 grayscale" 96 | # login_page_image_dark_filter = "contrast-200 blur-sm" 97 | 98 | # Specify a custom meta image url. 99 | # custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png" 100 | 101 | # Specify a custom build directory for the frontend. 102 | # This can be used to customize the frontend code. 103 | # Be careful: If this is a relative path, it should not start with a slash. 104 | # custom_build = "./public/build" 105 | 106 | # Specify optional one or more custom links in the header. 107 | # [[UI.header_links]] 108 | # name = "Issues" 109 | # icon_url = "https://avatars.githubusercontent.com/u/128686189?s=200&v=4" 110 | # url = "https://github.com/Chainlit/chainlit/issues" 111 | 112 | [meta] 113 | generated_by = "2.4.400" 114 | -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/README.md: -------------------------------------------------------------------------------- 1 | # Agentic RAG (Azure AI Search) 2 | 3 | The code first creates a sample index, then demonstrates 3 sample Agentic RAG strategies to query it: 4 | 1. Basic (No Reflection Included) 5 | 2. Single Step Reflection 6 | 3. Multi Step Reflection 7 | 8 | ## Prerequisites 9 | 10 | - An Azure subscription, with [access to Azure OpenAI](https://aka.ms/oai/access). 11 | 12 | - An [Azure AI Fondry Project](https://learn.microsoft.com/azure/ai-foundry/how-to/create-projects) 13 | 14 | - [Bing Grounding](https://learn.microsoft.com/azure/ai-services/agents/how-to/tools/bing-grounding) to access public data on the web 15 | 16 | - Azure AI Search, any version, but make sure search service capacity is sufficient for the workload. We recommend Basic or higher for this demo. 17 | 18 | - A deployment of the `text-embedding-3-large` embedding model in your Azure OpenAI service. As a naming convention, we name deployments after the model name: "text-embedding-3-large". 19 | 20 | - A deployment of a at least one [chat completion model](https://learn.microsoft.com/azure/ai-services/openai/how-to/chatgpt) in your Azure OpenAI service. You can try `gpt-4o-mini` for answer generation and query rewriting, and `gpt-4o` for evaluation and reflection. 21 | 22 | - Python (these instructions were tested with version 3.11.x) 23 | 24 | We used Visual Studio Code with the [Python extension](https://marketplace.visualstudio.com/items?itemName=ms-python.python) to test this sample. 25 | 26 | ### Setup Instructions 27 | 28 | 1. Ensure your python environment is setup. You might want to setup a [virtual environment](https://code.visualstudio.com/docs/python/environments) 29 | 1. Install requirements from `requirements.txt`: `pip install -r requirements.txt` 30 | 1. Copy `sample.env` to `.env` and fill out all environment variables. You do not need to provide a Search key if you are using [keyless](https://learn.microsoft.com/azure/search/search-security-rbac) authentication, but you must provide an Azure OpenAI key 31 | 1. Run the ingestion script to setup your sample index: `python ingest.py` 32 | 33 | ## Run the code 34 | 35 | 1. Run the sample app to interact with the sample index: `chainlit run app.py` 36 | 1. Navigate to `http://localhost:8000` to chat 37 | 1. You can load a sample conversation using the settings button 38 | 1. You can select which Agentic RAG strategy to use to answer any question 39 | 40 | ## Demo 41 | 42 | ![Demo](./media/demo.gif) -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/agents.py: -------------------------------------------------------------------------------- 1 | from openai import AsyncAzureOpenAI, pydantic_function_tool, AsyncStream 2 | from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessageParam, ChatCompletionUserMessageParam, ChatCompletionSystemMessageParam 3 | from azure.search.documents.aio import SearchClient 4 | from azure.search.documents.models import VectorizableTextQuery 5 | from azure.core.credentials_async import AsyncTokenCredential 6 | import chainlit as cl 7 | from chainlit.context import ChainlitContextException 8 | from typing import List, Any, Optional, Dict,Tuple 9 | from azure.ai.evaluation import GroundednessEvaluator, RelevanceEvaluator 10 | import prompty 11 | from azure.ai.projects.aio import AIProjectClient 12 | from azure.ai.projects.models import MessageRole, BingGroundingTool 13 | from tenacity import AsyncRetrying, wait_exponential, stop_after_attempt, retry_if_exception_type 14 | from copy import deepcopy 15 | import asyncio 16 | 17 | import json 18 | import functools 19 | from dataclasses import dataclass 20 | 21 | from dotenv import load_dotenv 22 | from models import ActionList, Action, ActionBase, SearchSources, Document, AnswerSources, AnswerAction, EvaluationAction, EvaluatorResponse, FilterDocumentsAction, IndexSearchAction, QueryRewriteAction, QuestionAction, ReflectionAction, StopAction, ToolChoiceReflection, WebSearchAction, WebCitation 23 | 24 | 25 | answer_prompt = prompty.load("prompts/chat_answer_question.prompty") 26 | query_rewrite_prompt = prompty.load("prompts/chat_query_rewrite.prompty") 27 | reflect_prompt = prompty.load("prompts/chat_reflect_answer.prompty") 28 | 29 | @dataclass 30 | class AppState: 31 | openai_client: AsyncAzureOpenAI 32 | search_client: SearchClient 33 | answer_generator_deployment: str 34 | query_rewriter_deployment: str 35 | reflection_deployment: str 36 | groundedness_evaluator: GroundednessEvaluator 37 | relevance_evaluator: RelevanceEvaluator 38 | ai_project_client: AIProjectClient 39 | bing_grounding_name: str 40 | bing_tool: Optional[BingGroundingTool] = None 41 | 42 | 43 | def running_in_cl() -> bool: 44 | try: 45 | cl.context.session.thread_id 46 | return True 47 | except ChainlitContextException: 48 | return False 49 | 50 | # Generate steps if running in chainlit, otherwise do nothing 51 | def step(func): 52 | @functools.wraps(func) 53 | async def step_wrapper(*args, **kwargs): 54 | cl_step: Optional[cl.Step] = None if not running_in_cl() else cl.Step(name=func.__name__) 55 | 56 | if cl_step: 57 | async with cl_step: 58 | cl_step.input = kwargs 59 | output = await func(*args, **kwargs) 60 | cl_step.output = output.model_dump() 61 | if isinstance(output, ActionBase) and output.render_during_step(): 62 | await output.render() 63 | return output 64 | else: 65 | return await func(*args, **kwargs) 66 | 67 | return step_wrapper 68 | 69 | 70 | @step 71 | async def rewrite_query(app_state: AppState, question: str, message_history: List[ChatCompletionMessageParam]) -> QueryRewriteAction: 72 | query_messages = prompty.prepare( 73 | query_rewrite_prompt, {"user_query": question, "past_messages": message_history} 74 | ) 75 | result = await app_state.openai_client.beta.chat.completions.parse( 76 | messages=query_messages, 77 | response_format=SearchSources, 78 | model=app_state.query_rewriter_deployment, 79 | temperature=0 80 | ) 81 | if parsed := result.choices[0].message.parsed: 82 | return QueryRewriteAction( 83 | query=question, message_history=message_history, rewritten_query=parsed.search_query 84 | ) 85 | 86 | return QueryRewriteAction( 87 | query=question, message_history=message_history, rewritten_query=None 88 | ) 89 | 90 | @step 91 | async def filter_documents(documents: List[Document], threshold: float = 2) -> FilterDocumentsAction: 92 | filtered_documents = [document for document in documents if document.score >= threshold] 93 | return FilterDocumentsAction( 94 | threshold=threshold, 95 | documents=documents, 96 | filtered_documents=filtered_documents 97 | ) 98 | 99 | @step 100 | async def index_search(app_state: AppState, question: str, message_history: List[ChatCompletionMessageParam]) -> IndexSearchAction: 101 | query = await rewrite_query(app_state, question, message_history) 102 | response = await app_state.search_client.search( 103 | search_text=query, 104 | vector_queries=[VectorizableTextQuery(text=query, k_nearest_neighbors=50, fields="embedding")], 105 | query_type="semantic", 106 | semantic_configuration_name="default", 107 | top=10, 108 | select=["content", "id", "sourcepage"] 109 | ) 110 | results = [Document(content=result["content"], id=result["id"], score=result["@search.reranker_score"], title=result["sourcepage"]) async for result in response] 111 | filtered_results = await filter_documents(results) 112 | return IndexSearchAction( 113 | query=question, 114 | rewritten_query=query.rewritten_query, 115 | documents=filtered_results.filtered_documents, 116 | ) 117 | 118 | @step 119 | async def web_search(app_state: AppState, query: str) -> WebSearchAction: 120 | # Initialize agent bing tool and add the connection id 121 | if not app_state.bing_tool: 122 | bing_connection = await app_state.ai_project_client.connections.get(app_state.bing_grounding_name) 123 | app_state.bing_tool = BingGroundingTool(connection_id=bing_connection.id) 124 | search_agent = await app_state.agents.create_agent( 125 | model="gpt-4o", 126 | name="my-web-assistant", 127 | instructions="You are a helpful assistant", 128 | tools=app_state.bing_tool.definitions, 129 | headers={"x-ms-enable-preview": "true"}, 130 | ) 131 | thread = await app_state.ai_project_client.agents.create_thread() 132 | await app_state.ai_project_client.agents.agents.create_message( 133 | thread_id=thread.id, 134 | role=MessageRole.USER, 135 | content="Search the web: " + query, 136 | ) 137 | async for run_attempt in AsyncRetrying(retry=retry_if_exception_type(ValueError), wait=wait_exponential(min=4, max=60), stop=stop_after_attempt(10)): 138 | with run_attempt: 139 | run = await app_state.ai_project_client.agents.create_and_process_run(thread_id=thread.id, agent_id=search_agent.id) 140 | if run.status == "failed" and run.last_error and run.last_error.get('code') == 'rate_limit_exceeded': 141 | raise ValueError("Rate limit exceeded. Please try again later.") 142 | 143 | if run.status == "failed": 144 | raise ValueError(run.last_error) 145 | 146 | response_messages = await app_state.ai_project_client.agents.list_messages(thread_id=thread.id) 147 | response_message = response_messages.get_last_message_by_role( 148 | MessageRole.AGENT 149 | ) 150 | if response_message.text_messages and len(response_message.text_messages) > 0: 151 | response_content = response_message.text_messages[0].text.value 152 | citations = [ WebCitation(name=annotation.text, url=annotation.url_citation.url, title=annotation.url_citation.title) for annotation in response_message.url_citation_annotations ] 153 | else: 154 | response_content = "" 155 | citations = [] 156 | 157 | 158 | return WebSearchAction( 159 | query=query, 160 | response_content=response_content, 161 | citations=citations, 162 | ) 163 | 164 | @step 165 | async def generate_answer(app_state: AppState, question: str, answer_sources: AnswerSources, previous_answer: Optional[str] = None, previous_answer_reflection: Optional[str] = None, previous_answer_evaluations: Optional[List[EvaluatorResponse]] = None ) -> AnswerAction: 166 | messages = prompty.prepare( 167 | answer_prompt, 168 | { 169 | "user_query": question, 170 | "context": answer_sources.create_context(), 171 | "previous_answer_reflection": previous_answer_reflection, 172 | "previous_answer": previous_answer, 173 | "previous_answer_evaluations": previous_answer_evaluations 174 | } 175 | ) 176 | 177 | stream: AsyncStream[ChatCompletionChunk] = await app_state.openai_client.chat.completions.create( 178 | messages=messages, 179 | stream=True, 180 | model=app_state.answer_generator_deployment, 181 | temperature=0.3 182 | ) 183 | 184 | answer_text = "" 185 | msg: Optional[cl.Message] = None if not running_in_cl() else cl.Message("") 186 | async for part in stream: 187 | if part.choices and (token := part.choices[0].delta.content or ""): 188 | answer_text += token 189 | if msg: 190 | await msg.stream_token(token) 191 | 192 | 193 | result = AnswerAction( 194 | question_text=question, 195 | answer_text=answer_text, 196 | answer_sources=answer_sources, 197 | ) 198 | 199 | if msg: 200 | msg.elements = result.create_pdf_citations() 201 | await msg.update() 202 | 203 | return result 204 | 205 | @step 206 | async def evaluate_answer(app_state: AppState, answer: AnswerAction, previous_reflection: Optional[ReflectionAction] = None) -> EvaluationAction: 207 | evaluator_context = answer.answer_sources.create_context() 208 | if previous_reflection: 209 | evaluator_context += f"\n\nPrevious Answer: {previous_reflection.answer_text}\n\nPrevious Answer Incorrect Reason: {previous_reflection.answer_reflection}" 210 | 211 | def eval_groundedness() -> Dict[str, Any]: 212 | return app_state.groundedness_evaluator( 213 | **{ 214 | "query": answer.question_text, 215 | "context": evaluator_context, 216 | "response": answer.answer_text 217 | } 218 | ) 219 | async_eval_groundedness = cl.make_async(eval_groundedness) 220 | 221 | def eval_relevance() -> Dict[str, Any]: 222 | return app_state.relevance_evaluator( 223 | **{ 224 | "query": answer.question_text, 225 | "response": answer.answer_text 226 | } 227 | ) 228 | async_eval_relevance = cl.make_async(eval_relevance) 229 | 230 | results = await asyncio.gather(async_eval_groundedness(), async_eval_relevance()) 231 | groundedness_result = results[0] 232 | relevance_result = results[1] 233 | 234 | results = [ 235 | EvaluatorResponse( 236 | type="relevance", 237 | score=relevance_result['relevance'], 238 | explanation=relevance_result['relevance_reason'] 239 | ), 240 | EvaluatorResponse( 241 | type="groundedness", 242 | score=groundedness_result['groundedness'], 243 | explanation=groundedness_result['groundedness_reason'] 244 | ) 245 | ] 246 | 247 | return EvaluationAction( 248 | evaluations=results, 249 | answer=answer 250 | ) 251 | 252 | 253 | @step 254 | async def reflect_answer(app_state: AppState, action_list: ActionList) -> ReflectionAction: 255 | last_answer_action = action_list.get_last_answer() 256 | last_query = action_list.get_last_query() 257 | last_evaluation = action_list.get_last_evaluation() 258 | assert last_answer_action != None 259 | assert last_query != None 260 | assert last_evaluation != None 261 | reflect_messages = prompty.prepare( 262 | reflect_prompt, 263 | { 264 | "previous_answer": last_answer_action.answer_text, 265 | "previous_sources": last_answer_action.answer_sources.create_context(), 266 | "previous_query": last_query, 267 | "previous_answer_evlauation": "\n".join(f"{evaluation.type}: {evaluation.score}, {evaluation.explanation}" for evaluation in last_evaluation.evaluations) 268 | } 269 | ) 270 | result: ChatCompletion = await app_state.openai_client.chat.completions.create( 271 | messages=reflect_messages, 272 | tools=[pydantic_function_tool(ToolChoiceReflection)], 273 | model=app_state.reflection_deployment 274 | ) 275 | action = ReflectionAction( 276 | answer_text="I don't know", 277 | answer_reflection="I don't know", 278 | follow_up_action="stop", 279 | follow_up_action_reason="I don't know what to do next.", 280 | actions=action_list.actions 281 | ) 282 | 283 | if result.choices[0].message.tool_calls: 284 | tool_call = result.choices[0].message.tool_calls[0] 285 | tool_reflection = json.loads(tool_call.function.arguments) 286 | follow_up_action_str = "stop" 287 | if "index_query" in tool_reflection["follow_up_action"]: 288 | follow_up_action_str = "index_search" 289 | next_query = tool_reflection["follow_up_action"]["index_query"] 290 | elif "web_query" in tool_reflection["follow_up_action"]: 291 | follow_up_action_str = "web_search" 292 | next_query = tool_reflection["follow_up_action"]["web_query"] 293 | action = ReflectionAction( 294 | answer_text=last_answer_action.answer_text, 295 | answer_reflection=tool_reflection["answer_reflection"], 296 | follow_up_action=follow_up_action_str, 297 | follow_up_action_reason=tool_reflection["follow_up_action_reason"], 298 | next_query=next_query, 299 | actions=action_list.actions 300 | ) 301 | 302 | return action 303 | 304 | 305 | async def index_search_only(app_state: AppState, message: str, message_history: Optional[List[ChatCompletionMessageParam]] = None) -> ActionList: 306 | question_action = QuestionAction(question_text=message) 307 | index_search_action = await index_search(app_state, message, message_history) 308 | answer_action = await generate_answer(app_state, message, AnswerSources.from_index_search(message_history, index_search_action)) 309 | 310 | return ActionList(actions=[question_action, index_search_action, answer_action]) 311 | 312 | async def reflect_next_action(app_state: AppState, reflection: ReflectionAction, answer_sources: AnswerSources) -> Tuple[Action, AnswerSources]: 313 | if reflection.follow_up_action == "index_search" and reflection.next_query: 314 | index_action = await index_search(app_state, reflection.next_query, answer_sources.message_history) 315 | next_sources = answer_sources.add_index_search(index_action) 316 | return (index_action, next_sources) 317 | elif reflection.follow_up_action == "web_search" and reflection.next_query: 318 | web_search_action = await web_search(app_state, reflection.next_query) 319 | next_sources = answer_sources.add_web_search(web_search_action) 320 | return (web_search_action, next_sources) 321 | elif reflection.follow_up_action == "stop": 322 | return (StopAction(), answer_sources) 323 | 324 | async def reflection_single_step(app_state: AppState, message: str, message_history: Optional[List[ChatCompletionMessageParam]] = None) -> ActionList: 325 | question_action = QuestionAction(question_text=message) 326 | index_search_action = await index_search(app_state, message, message_history) 327 | answer_action = await generate_answer(app_state, message, AnswerSources.from_index_search(message_history, index_search_action)) 328 | evaluation_action = await evaluate_answer(app_state, answer_action) 329 | action_list = ActionList(actions=[question_action, index_search_action, answer_action, evaluation_action ]) 330 | if any(evaluation.score < 4 for evaluation in evaluation_action.evaluations): 331 | reflection_action = await reflect_answer(app_state, action_list) 332 | next_action, next_sources = await reflect_next_action(app_state, reflection_action, answer_action.answer_sources) 333 | final_answer_action = await generate_answer(app_state, message, next_sources, previous_answer=answer_action.answer_text, previous_answer_reflection=reflection_action.answer_reflection, previous_answer_evaluations=evaluation_action.evaluations) 334 | final_evaluation_action = await evaluate_answer(app_state, final_answer_action) 335 | 336 | action_list.actions.extend([reflection_action, next_action, final_answer_action, final_evaluation_action]) 337 | 338 | return action_list 339 | 340 | async def reflection_multi_step(app_state: AppState, message: str, message_history: Optional[List[ChatCompletionMessageParam]]) -> ActionList: 341 | question_action = QuestionAction(question_text=message) 342 | index_search_action = await index_search(app_state, message, message_history) 343 | answer_action = await generate_answer(app_state, message, AnswerSources.from_index_search(message_history, index_search_action)) 344 | evaluation_action = await evaluate_answer(app_state, answer_action) 345 | action_list = ActionList(actions=[question_action, index_search_action, answer_action, evaluation_action ]) 346 | max_turns = 3 347 | turn = 1 348 | while any(evaluation.score < 4 for evaluation in evaluation_action.evaluations): 349 | reflection_action = await reflect_answer(app_state, action_list) 350 | next_action, next_sources = await reflect_next_action(app_state, reflection_action, answer_action.answer_sources) 351 | next_answer_action = await generate_answer(app_state, message, next_sources, previous_answer=answer_action.answer_text, previous_answer_reflection=reflection_action.answer_reflection, previous_answer_evaluations=evaluation_action.evaluations) 352 | evaluation_action = await evaluate_answer(app_state, next_answer_action, reflection_action) 353 | action_list.actions.extend([reflection_action, next_action, next_answer_action, evaluation_action]) 354 | 355 | if reflection_action.type == "stop": 356 | break 357 | turn += 1 358 | if turn > max_turns: 359 | break 360 | 361 | return action_list 362 | 363 | -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/app.py: -------------------------------------------------------------------------------- 1 | import chainlit as cl 2 | 3 | import os 4 | from pathlib import Path 5 | 6 | from agents import ActionList, AppState, index_search_only, reflection_single_step, reflection_multi_step 7 | from azure.ai.evaluation import GroundednessEvaluator, RelevanceEvaluator 8 | from azure.core.credentials import AzureKeyCredential 9 | from azure.identity.aio import AzureCliCredential, get_bearer_token_provider 10 | from dotenv import load_dotenv 11 | from openai import AsyncAzureOpenAI 12 | from azure.search.documents.aio import SearchClient 13 | from azure.ai.projects.aio import AIProjectClient 14 | 15 | # Load environment variables 16 | load_dotenv() 17 | 18 | credential = AzureCliCredential() 19 | token_provider = get_bearer_token_provider(credential, "https://cognitiveservices.azure.com/.default") 20 | 21 | evaluator_model_config = { 22 | "azure_endpoint": os.environ["AZURE_OPENAI_ENDPOINT"], # Replace with your Azure OpenAI endpoint 23 | "azure_deployment": os.environ["AZURE_OPENAI_EVALUATOR_DEPLOYMENT"], 24 | "api_version": os.environ["AZURE_OPENAI_API_VERSION"], 25 | "api_key": os.environ["AZURE_OPENAI_API_KEY"], 26 | } 27 | 28 | app_state = AppState( 29 | openai_client = AsyncAzureOpenAI( 30 | api_version=os.environ["AZURE_OPENAI_API_VERSION"], 31 | azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"], 32 | azure_ad_token_provider=token_provider 33 | ), 34 | search_client = SearchClient(endpoint=os.environ["AZURE_SEARCH_ENDPOINT"], index_name=os.environ["AZURE_SEARCH_INDEX"], credential=AzureKeyCredential(os.getenv("AZURE_SEARCH_KEY")) if os.getenv("AZURE_SEARCH_KEY") else credential), 35 | answer_generator_deployment = os.environ["AZURE_OPENAI_GENERATOR_DEPLOYMENT"], 36 | query_rewriter_deployment = os.environ["AZURE_OPENAI_REWRITER_DEPLOYMENT"], 37 | reflection_deployment = os.environ["AZURE_OPENAI_REFLECTION_DEPLOYMENT"], 38 | groundedness_evaluator = GroundednessEvaluator(evaluator_model_config), 39 | relevance_evaluator = RelevanceEvaluator(evaluator_model_config), 40 | ai_project_client = AIProjectClient.from_connection_string(os.environ["AI_PROJECT_CONN_STR"], credential), 41 | bing_grounding_name = os.environ["BING_GROUNDING_NAME"], 42 | ) 43 | 44 | basic_chat_type = 'Basic' 45 | single_reflection_type = 'Single Step Reflection' 46 | multi_reflection_type = 'Multi Step Reflection' 47 | 48 | commands = [ 49 | {"id": basic_chat_type, "icon": "move-right", "description": "Use basic RAG strategy", "button": True}, 50 | {"id": single_reflection_type, "icon": "corner-down-right", "description": "Use single step reflection RAG strategy", "button": True}, 51 | {"id": multi_reflection_type, "icon": "iteration-ccw", "description": "Use multi step reflection RAG strategy", "button": True} 52 | ] 53 | 54 | chat_type = multi_reflection_type 55 | 56 | # Instrument the OpenAI client 57 | cl.instrument_openai() 58 | 59 | @cl.on_chat_start 60 | async def start(): 61 | await cl.context.emitter.set_commands(commands) 62 | await cl.ChatSettings( 63 | [ 64 | cl.input_widget.Select( 65 | id="ChatType", 66 | label="Chat Type", 67 | values=[basic_chat_type, single_reflection_type, multi_reflection_type], 68 | initial_index=2 69 | ), 70 | cl.input_widget.Select( 71 | id="PreviousConversation", 72 | label="Load a previous conversation", 73 | values=[os.path.splitext(os.path.basename(k))[0] for k in Path("examples").glob("*.json")], 74 | initial_index=None 75 | ) 76 | ] 77 | ).send() 78 | cl.user_session.set("actions", ActionList(actions=[])) 79 | 80 | @cl.on_settings_update 81 | async def update_settings(settings): 82 | global chat_type 83 | chat_type = settings["ChatType"] 84 | 85 | file = None 86 | if conversation := settings["PreviousConversation"]: 87 | file = f"examples/{conversation}.json" 88 | 89 | if file: 90 | with open(file, "r", encoding="utf-8") as f: 91 | action_list = ActionList.model_validate_json(f.read()) 92 | 93 | for action in action_list.actions: 94 | if action.create_step(): 95 | async with cl.Step(action.step_name()) as step: 96 | step.input = action.get_input_props() 97 | step.output = action.get_output_props() 98 | await action.render() 99 | else: 100 | await action.render() 101 | 102 | session_action_list: ActionList = cl.user_session.get("actions") 103 | session_action_list.actions.extend(action_list.actions) 104 | 105 | @cl.on_message 106 | async def on_message(msg: cl.Message): 107 | question = msg.content 108 | 109 | message_chat_type = chat_type 110 | if msg.command: 111 | message_chat_type = msg.command 112 | 113 | session_action_list: ActionList = cl.user_session.get("actions") 114 | last_answer_action = session_action_list.get_last_answer() 115 | # If there was a last answer action, append its message history to the new question 116 | message_history = [] 117 | if last_answer_action: 118 | last_answer_action.append_qa_message_history() 119 | message_history = last_answer_action.answer_sources.message_history 120 | 121 | if message_chat_type == basic_chat_type: 122 | action_list = await index_search_only(app_state, question, message_history) 123 | elif message_chat_type == single_reflection_type: 124 | action_list = await reflection_single_step(app_state, msg.content, message_history) 125 | elif message_chat_type == multi_reflection_type: 126 | action_list = await reflection_multi_step(app_state, msg.content, message_history) 127 | else: 128 | action_list = ActionList(actions=[]) 129 | 130 | session_action_list.actions.extend(action_list.actions) 131 | 132 | starters = [ 133 | "What is included in my Northwind Health Plus plan that is not in standard?", 134 | "Who ensures the compliance of the Northwind Health Plus plan with state regulations?", 135 | "What is the deductible for prescription drugs provided by Northwind Health?" 136 | ] 137 | @cl.set_starters 138 | async def set_starters(): 139 | return [ 140 | cl.Starter( 141 | label=k, 142 | message=k, 143 | ) 144 | for k in starters 145 | ] -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/chainlit.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey Bonus - Agentic RAG/sample/chainlit.md -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/data/Benefit_Options.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey Bonus - Agentic RAG/sample/data/Benefit_Options.pdf -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/data/Northwind_Health_Plus_Benefits_Details.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey Bonus - Agentic RAG/sample/data/Northwind_Health_Plus_Benefits_Details.pdf -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/data/Northwind_Standard_Benefits_Details.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey Bonus - Agentic RAG/sample/data/Northwind_Standard_Benefits_Details.pdf -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/examples/Web Search Reflection.json: -------------------------------------------------------------------------------- 1 | { 2 | "actions": [ 3 | { 4 | "type": "question", 5 | "question_text": "How do hospitals ensure patients receive emergency care?" 6 | }, 7 | { 8 | "type": "index_search", 9 | "documents": [ 10 | { 11 | "content": "\n· Have your insurance card on hand: Bring your insurance card with you to the visit, as you may need to provide it to the facility.\n· Bring your medical records: If you have any relevant medical records, bring them with you to the facility. This will help the provider make a more informed diagnosis.\n· Know when to go to the emergency room: If you are experiencing a medical emergency, call 911 or go to the nearest emergency room for treatment.\nWe understand that receiving urgent care can be a stressful situation. At Northwind Health Plus, we strive to make the process as easy and stress-free as possible. If you have any questions about your coverage, please reach out to us so we can help.\nVirtual Care COVERED SERVICES: Virtual Care\nAt Contoso, we understand that taking time off to go to the doctor's office isn't always possible. That's why we've partnered with Northwind Health to provide access to virtual care services. With Northwind Health Plus, you can get the care you need from the comfort of your own home, office, or other location.", 12 | "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-142", 13 | "score": 2.1660687923431396, 14 | "title": "Northwind_Health_Plus_Benefits_Details.pdf#page=60" 15 | } 16 | ], 17 | "query": "How do hospitals ensure patients receive emergency care?", 18 | "rewritten_query": "hospitals emergency care patient access procedures" 19 | }, 20 | { 21 | "type": "answer", 22 | "question_text": "How do hospitals ensure patients receive emergency care?", 23 | "answer_text": "Hospitals ensure patients receive emergency care by advising them to know when to go to the emergency room, which is when experiencing a medical emergency. Patients are encouraged to call 911 or go to the nearest emergency room for treatment. Additionally, it's important for patients to bring their insurance card and any relevant medical records to help providers make informed diagnoses. Northwind Health Plus aims to make the process as easy and stress-free as possible for patients seeking urgent care [Northwind_Health_Plus_Benefits_Details.pdf#page=60].", 24 | "answer_sources": { 25 | "message_history": [], 26 | "text_sources": [ 27 | { 28 | "content": "\n· Have your insurance card on hand: Bring your insurance card with you to the visit, as you may need to provide it to the facility.\n· Bring your medical records: If you have any relevant medical records, bring them with you to the facility. This will help the provider make a more informed diagnosis.\n· Know when to go to the emergency room: If you are experiencing a medical emergency, call 911 or go to the nearest emergency room for treatment.\nWe understand that receiving urgent care can be a stressful situation. At Northwind Health Plus, we strive to make the process as easy and stress-free as possible. If you have any questions about your coverage, please reach out to us so we can help.\nVirtual Care COVERED SERVICES: Virtual Care\nAt Contoso, we understand that taking time off to go to the doctor's office isn't always possible. That's why we've partnered with Northwind Health to provide access to virtual care services. With Northwind Health Plus, you can get the care you need from the comfort of your own home, office, or other location.", 29 | "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-142", 30 | "score": 2.1660687923431396, 31 | "title": "Northwind_Health_Plus_Benefits_Details.pdf#page=60" 32 | } 33 | ], 34 | "public_sources": null, 35 | "public_citations": null 36 | } 37 | }, 38 | { 39 | "type": "evaluation", 40 | "evaluations": [ 41 | { 42 | "type": "relevance", 43 | "score": 3, 44 | "explanation": "The response provides partial information relevant to the query but omits key details about hospital systems and processes for ensuring emergency care, making it incomplete." 45 | }, 46 | { 47 | "type": "groundedness", 48 | "score": 5, 49 | "explanation": "The RESPONSE is fully grounded in the CONTEXT, directly answering the QUERY with accurate and complete information." 50 | } 51 | ], 52 | "answer": { 53 | "type": "answer", 54 | "question_text": "How do hospitals ensure patients receive emergency care?", 55 | "answer_text": "Hospitals ensure patients receive emergency care by advising them to know when to go to the emergency room, which is when experiencing a medical emergency. Patients are encouraged to call 911 or go to the nearest emergency room for treatment. Additionally, it's important for patients to bring their insurance card and any relevant medical records to help providers make informed diagnoses. Northwind Health Plus aims to make the process as easy and stress-free as possible for patients seeking urgent care [Northwind_Health_Plus_Benefits_Details.pdf#page=60].", 56 | "answer_sources": { 57 | "message_history": [], 58 | "text_sources": [ 59 | { 60 | "content": "\n· Have your insurance card on hand: Bring your insurance card with you to the visit, as you may need to provide it to the facility.\n· Bring your medical records: If you have any relevant medical records, bring them with you to the facility. This will help the provider make a more informed diagnosis.\n· Know when to go to the emergency room: If you are experiencing a medical emergency, call 911 or go to the nearest emergency room for treatment.\nWe understand that receiving urgent care can be a stressful situation. At Northwind Health Plus, we strive to make the process as easy and stress-free as possible. If you have any questions about your coverage, please reach out to us so we can help.\nVirtual Care COVERED SERVICES: Virtual Care\nAt Contoso, we understand that taking time off to go to the doctor's office isn't always possible. That's why we've partnered with Northwind Health to provide access to virtual care services. With Northwind Health Plus, you can get the care you need from the comfort of your own home, office, or other location.", 61 | "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-142", 62 | "score": 2.1660687923431396, 63 | "title": "Northwind_Health_Plus_Benefits_Details.pdf#page=60" 64 | } 65 | ], 66 | "public_sources": null, 67 | "public_citations": null 68 | } 69 | } 70 | }, 71 | { 72 | "type": "reflection", 73 | "answer_text": "Hospitals ensure patients receive emergency care by advising them to know when to go to the emergency room, which is when experiencing a medical emergency. Patients are encouraged to call 911 or go to the nearest emergency room for treatment. Additionally, it's important for patients to bring their insurance card and any relevant medical records to help providers make informed diagnoses. Northwind Health Plus aims to make the process as easy and stress-free as possible for patients seeking urgent care [Northwind_Health_Plus_Benefits_Details.pdf#page=60].", 74 | "answer_reflection": "The answer incorrectly implies that the internal source fully addressed the procedures for emergency care access in hospitals. However, the source only briefly outlines steps like bringing an insurance card and medical records, and knowing when to visit the emergency room. It lacked specific procedural details on hospital access and intake in emergencies, such as triage protocols or hospital-specific instructions.", 75 | "follow_up_action": "web_search", 76 | "follow_up_action_reason": "The source material did not sufficiently cover detailed emergency care access procedures at hospitals. To correct the answer, I should locate more comprehensive, publicly accessible information addressing this topic.", 77 | "next_query": "hospitals emergency care procedures for patient intake", 78 | "actions": [ 79 | { 80 | "type": "question", 81 | "question_text": "How do hospitals ensure patients receive emergency care?" 82 | }, 83 | { 84 | "type": "index_search", 85 | "documents": [ 86 | { 87 | "content": "\n· Have your insurance card on hand: Bring your insurance card with you to the visit, as you may need to provide it to the facility.\n· Bring your medical records: If you have any relevant medical records, bring them with you to the facility. This will help the provider make a more informed diagnosis.\n· Know when to go to the emergency room: If you are experiencing a medical emergency, call 911 or go to the nearest emergency room for treatment.\nWe understand that receiving urgent care can be a stressful situation. At Northwind Health Plus, we strive to make the process as easy and stress-free as possible. If you have any questions about your coverage, please reach out to us so we can help.\nVirtual Care COVERED SERVICES: Virtual Care\nAt Contoso, we understand that taking time off to go to the doctor's office isn't always possible. That's why we've partnered with Northwind Health to provide access to virtual care services. With Northwind Health Plus, you can get the care you need from the comfort of your own home, office, or other location.", 88 | "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-142", 89 | "score": 2.1660687923431396, 90 | "title": "Northwind_Health_Plus_Benefits_Details.pdf#page=60" 91 | } 92 | ], 93 | "query": "How do hospitals ensure patients receive emergency care?", 94 | "rewritten_query": "hospitals emergency care patient access procedures" 95 | }, 96 | { 97 | "type": "answer", 98 | "question_text": "How do hospitals ensure patients receive emergency care?", 99 | "answer_text": "Hospitals ensure patients receive emergency care by advising them to know when to go to the emergency room, which is when experiencing a medical emergency. Patients are encouraged to call 911 or go to the nearest emergency room for treatment. Additionally, it's important for patients to bring their insurance card and any relevant medical records to help providers make informed diagnoses. Northwind Health Plus aims to make the process as easy and stress-free as possible for patients seeking urgent care [Northwind_Health_Plus_Benefits_Details.pdf#page=60].", 100 | "answer_sources": { 101 | "message_history": [], 102 | "text_sources": [ 103 | { 104 | "content": "\n· Have your insurance card on hand: Bring your insurance card with you to the visit, as you may need to provide it to the facility.\n· Bring your medical records: If you have any relevant medical records, bring them with you to the facility. This will help the provider make a more informed diagnosis.\n· Know when to go to the emergency room: If you are experiencing a medical emergency, call 911 or go to the nearest emergency room for treatment.\nWe understand that receiving urgent care can be a stressful situation. At Northwind Health Plus, we strive to make the process as easy and stress-free as possible. If you have any questions about your coverage, please reach out to us so we can help.\nVirtual Care COVERED SERVICES: Virtual Care\nAt Contoso, we understand that taking time off to go to the doctor's office isn't always possible. That's why we've partnered with Northwind Health to provide access to virtual care services. With Northwind Health Plus, you can get the care you need from the comfort of your own home, office, or other location.", 105 | "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-142", 106 | "score": 2.1660687923431396, 107 | "title": "Northwind_Health_Plus_Benefits_Details.pdf#page=60" 108 | } 109 | ], 110 | "public_sources": null, 111 | "public_citations": null 112 | } 113 | }, 114 | { 115 | "type": "evaluation", 116 | "evaluations": [ 117 | { 118 | "type": "relevance", 119 | "score": 3, 120 | "explanation": "The response provides partial information relevant to the query but omits key details about hospital systems and processes for ensuring emergency care, making it incomplete." 121 | }, 122 | { 123 | "type": "groundedness", 124 | "score": 5, 125 | "explanation": "The RESPONSE is fully grounded in the CONTEXT, directly answering the QUERY with accurate and complete information." 126 | } 127 | ], 128 | "answer": { 129 | "type": "answer", 130 | "question_text": "How do hospitals ensure patients receive emergency care?", 131 | "answer_text": "Hospitals ensure patients receive emergency care by advising them to know when to go to the emergency room, which is when experiencing a medical emergency. Patients are encouraged to call 911 or go to the nearest emergency room for treatment. Additionally, it's important for patients to bring their insurance card and any relevant medical records to help providers make informed diagnoses. Northwind Health Plus aims to make the process as easy and stress-free as possible for patients seeking urgent care [Northwind_Health_Plus_Benefits_Details.pdf#page=60].", 132 | "answer_sources": { 133 | "message_history": [], 134 | "text_sources": [ 135 | { 136 | "content": "\n· Have your insurance card on hand: Bring your insurance card with you to the visit, as you may need to provide it to the facility.\n· Bring your medical records: If you have any relevant medical records, bring them with you to the facility. This will help the provider make a more informed diagnosis.\n· Know when to go to the emergency room: If you are experiencing a medical emergency, call 911 or go to the nearest emergency room for treatment.\nWe understand that receiving urgent care can be a stressful situation. At Northwind Health Plus, we strive to make the process as easy and stress-free as possible. If you have any questions about your coverage, please reach out to us so we can help.\nVirtual Care COVERED SERVICES: Virtual Care\nAt Contoso, we understand that taking time off to go to the doctor's office isn't always possible. That's why we've partnered with Northwind Health to provide access to virtual care services. With Northwind Health Plus, you can get the care you need from the comfort of your own home, office, or other location.", 137 | "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-142", 138 | "score": 2.1660687923431396, 139 | "title": "Northwind_Health_Plus_Benefits_Details.pdf#page=60" 140 | } 141 | ], 142 | "public_sources": null, 143 | "public_citations": null 144 | } 145 | } 146 | } 147 | ] 148 | }, 149 | { 150 | "type": "web_search", 151 | "query": "hospitals emergency care procedures for patient intake", 152 | "response_content": "Emergency care procedures in hospitals typically follow a structured process to ensure the most critical patients receive care first. The primary steps include:\n\n1. **Triage**: Initial assessment to prioritize patients by severity of their condition using a triage scale (e.g., Emergency Severity Index). \n2. **Registration**: Patients provide identification, insurance details, and medical history.\n3. **Medical Evaluation and Testing**: A healthcare provider conducts examinations and may order diagnostic tests (e.g., blood tests, X-rays).\n4. **Treatment and Monitoring**: Treatment includes medications, procedures, or observation.\n5. **Discharge or Admission**: Based on evaluation, patients are either discharged with instructions or admitted for further care【9†source】【7†source】.", 153 | "citations": [ 154 | { 155 | "name": "【9†source】", 156 | "url": "https://www.dignityhealth.org/conditions-and-treatments/emergency", 157 | "title": "Emergency Conditions Symptoms & Causes - Dignity Health" 158 | }, 159 | { 160 | "name": "【7†source】", 161 | "url": "https://www.acep.org/siteassets/new-pdfs/policy-statements/standardized-protocols-for-optimizing-emergency-department-care.pdf", 162 | "title": "Microsoft Word - Standardized Protocols for Optimizing Emergency ... - ACEP" 163 | } 164 | ] 165 | }, 166 | { 167 | "type": "answer", 168 | "question_text": "How do hospitals ensure patients receive emergency care?", 169 | "answer_text": "Hospitals ensure patients receive emergency care through a structured process that includes several key steps:\n\n1. **Triage**: An initial assessment to prioritize patients based on the severity of their condition.\n2. **Registration**: Patients provide identification, insurance details, and medical history.\n3. **Medical Evaluation and Testing**: Healthcare providers conduct examinations and may order diagnostic tests.\n4. **Treatment and Monitoring**: Patients receive necessary treatments, which may include medications or procedures.\n5. **Discharge or Admission**: Patients are either discharged with instructions or admitted for further care based on their evaluation [Emergency Conditions Symptoms & Causes - Dignity Health](https://www.dignityhealth.org/conditions-and-treatments/emergency) [Microsoft Word - Standardized Protocols for Optimizing Emergency ... - ACEP](https://www.acep.org/siteassets/new-pdfs/policy-statements/standardized-protocols-for-optimizing-emergency-department-care.pdf).\n\nAdditionally, patients are advised to bring their insurance card and any relevant medical records to facilitate the process and ensure they receive appropriate care [Northwind_Health_Plus_Benefits_Details.pdf#page=60].", 170 | "answer_sources": { 171 | "message_history": [], 172 | "text_sources": [ 173 | { 174 | "content": "\n· Have your insurance card on hand: Bring your insurance card with you to the visit, as you may need to provide it to the facility.\n· Bring your medical records: If you have any relevant medical records, bring them with you to the facility. This will help the provider make a more informed diagnosis.\n· Know when to go to the emergency room: If you are experiencing a medical emergency, call 911 or go to the nearest emergency room for treatment.\nWe understand that receiving urgent care can be a stressful situation. At Northwind Health Plus, we strive to make the process as easy and stress-free as possible. If you have any questions about your coverage, please reach out to us so we can help.\nVirtual Care COVERED SERVICES: Virtual Care\nAt Contoso, we understand that taking time off to go to the doctor's office isn't always possible. That's why we've partnered with Northwind Health to provide access to virtual care services. With Northwind Health Plus, you can get the care you need from the comfort of your own home, office, or other location.", 175 | "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-142", 176 | "score": 2.1660687923431396, 177 | "title": "Northwind_Health_Plus_Benefits_Details.pdf#page=60" 178 | } 179 | ], 180 | "public_sources": "Emergency care procedures in hospitals typically follow a structured process to ensure the most critical patients receive care first. The primary steps include:\n\n1. **Triage**: Initial assessment to prioritize patients by severity of their condition using a triage scale (e.g., Emergency Severity Index). \n2. **Registration**: Patients provide identification, insurance details, and medical history.\n3. **Medical Evaluation and Testing**: A healthcare provider conducts examinations and may order diagnostic tests (e.g., blood tests, X-rays).\n4. **Treatment and Monitoring**: Treatment includes medications, procedures, or observation.\n5. **Discharge or Admission**: Based on evaluation, patients are either discharged with instructions or admitted for further care【9†source】【7†source】.", 181 | "public_citations": [ 182 | { 183 | "name": "【9†source】", 184 | "url": "https://www.dignityhealth.org/conditions-and-treatments/emergency", 185 | "title": "Emergency Conditions Symptoms & Causes - Dignity Health" 186 | }, 187 | { 188 | "name": "【7†source】", 189 | "url": "https://www.acep.org/siteassets/new-pdfs/policy-statements/standardized-protocols-for-optimizing-emergency-department-care.pdf", 190 | "title": "Microsoft Word - Standardized Protocols for Optimizing Emergency ... - ACEP" 191 | } 192 | ] 193 | } 194 | }, 195 | { 196 | "type": "evaluation", 197 | "evaluations": [ 198 | { 199 | "type": "relevance", 200 | "score": 5, 201 | "explanation": "The RESPONSE fully addresses the QUERY with accurate and complete information, while also providing additional relevant insights to enhance understanding." 202 | }, 203 | { 204 | "type": "groundedness", 205 | "score": 5, 206 | "explanation": "The RESPONSE is fully grounded in the CONTEXT, accurately and completely answering the QUERY by combining relevant details from both internal and public sources." 207 | } 208 | ], 209 | "answer": { 210 | "type": "answer", 211 | "question_text": "How do hospitals ensure patients receive emergency care?", 212 | "answer_text": "Hospitals ensure patients receive emergency care through a structured process that includes several key steps:\n\n1. **Triage**: An initial assessment to prioritize patients based on the severity of their condition.\n2. **Registration**: Patients provide identification, insurance details, and medical history.\n3. **Medical Evaluation and Testing**: Healthcare providers conduct examinations and may order diagnostic tests.\n4. **Treatment and Monitoring**: Patients receive necessary treatments, which may include medications or procedures.\n5. **Discharge or Admission**: Patients are either discharged with instructions or admitted for further care based on their evaluation [Emergency Conditions Symptoms & Causes - Dignity Health](https://www.dignityhealth.org/conditions-and-treatments/emergency) [Microsoft Word - Standardized Protocols for Optimizing Emergency ... - ACEP](https://www.acep.org/siteassets/new-pdfs/policy-statements/standardized-protocols-for-optimizing-emergency-department-care.pdf).\n\nAdditionally, patients are advised to bring their insurance card and any relevant medical records to facilitate the process and ensure they receive appropriate care [Northwind_Health_Plus_Benefits_Details.pdf#page=60].", 213 | "answer_sources": { 214 | "message_history": [], 215 | "text_sources": [ 216 | { 217 | "content": "\n· Have your insurance card on hand: Bring your insurance card with you to the visit, as you may need to provide it to the facility.\n· Bring your medical records: If you have any relevant medical records, bring them with you to the facility. This will help the provider make a more informed diagnosis.\n· Know when to go to the emergency room: If you are experiencing a medical emergency, call 911 or go to the nearest emergency room for treatment.\nWe understand that receiving urgent care can be a stressful situation. At Northwind Health Plus, we strive to make the process as easy and stress-free as possible. If you have any questions about your coverage, please reach out to us so we can help.\nVirtual Care COVERED SERVICES: Virtual Care\nAt Contoso, we understand that taking time off to go to the doctor's office isn't always possible. That's why we've partnered with Northwind Health to provide access to virtual care services. With Northwind Health Plus, you can get the care you need from the comfort of your own home, office, or other location.", 218 | "id": "file-Northwind_Health_Plus_Benefits_Details_pdf-4E6F72746877696E645F4865616C74685F506C75735F42656E65666974735F44657461696C732E706466-page-142", 219 | "score": 2.1660687923431396, 220 | "title": "Northwind_Health_Plus_Benefits_Details.pdf#page=60" 221 | } 222 | ], 223 | "public_sources": "Emergency care procedures in hospitals typically follow a structured process to ensure the most critical patients receive care first. The primary steps include:\n\n1. **Triage**: Initial assessment to prioritize patients by severity of their condition using a triage scale (e.g., Emergency Severity Index). \n2. **Registration**: Patients provide identification, insurance details, and medical history.\n3. **Medical Evaluation and Testing**: A healthcare provider conducts examinations and may order diagnostic tests (e.g., blood tests, X-rays).\n4. **Treatment and Monitoring**: Treatment includes medications, procedures, or observation.\n5. **Discharge or Admission**: Based on evaluation, patients are either discharged with instructions or admitted for further care【9†source】【7†source】.", 224 | "public_citations": [ 225 | { 226 | "name": "【9†source】", 227 | "url": "https://www.dignityhealth.org/conditions-and-treatments/emergency", 228 | "title": "Emergency Conditions Symptoms & Causes - Dignity Health" 229 | }, 230 | { 231 | "name": "【7†source】", 232 | "url": "https://www.acep.org/siteassets/new-pdfs/policy-statements/standardized-protocols-for-optimizing-emergency-department-care.pdf", 233 | "title": "Microsoft Word - Standardized Protocols for Optimizing Emergency ... - ACEP" 234 | } 235 | ] 236 | } 237 | } 238 | } 239 | ] 240 | } -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/ingest.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import asyncio 4 | from azure.identity.aio import DefaultAzureCredential 5 | from azure.search.documents.aio import SearchClient 6 | from azure.search.documents.indexes.models import SearchIndex, AzureOpenAIVectorizer, AzureOpenAIVectorizerParameters 7 | from azure.search.documents.indexes.aio import SearchIndexClient 8 | from azure.core.credentials import AzureKeyCredential 9 | from dotenv import load_dotenv 10 | 11 | async def main(): 12 | load_dotenv() 13 | 14 | credential = AzureKeyCredential(os.getenv("AZURE_SEARCH_KEY")) if os.getenv("AZURE_SEARCH_KEY") else DefaultAzureCredential() 15 | index_client = SearchIndexClient(endpoint=os.environ["AZURE_SEARCH_ENDPOINT"], credential=credential) 16 | with open("sample-index/schema.json") as f: 17 | index = json.load(f) 18 | search_index = SearchIndex.deserialize(index) 19 | search_index.name = os.environ["AZURE_SEARCH_INDEX"] 20 | search_index.vector_search.vectorizers.append( 21 | AzureOpenAIVectorizer( 22 | vectorizer_name="vectorizer", 23 | parameters=AzureOpenAIVectorizerParameters( 24 | resource_url=os.environ["AZURE_OPENAI_ENDPOINT"], 25 | deployment_name=os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"], 26 | model_name="text-embedding-3-large", 27 | api_key=os.environ["AZURE_OPENAI_API_KEY"] 28 | ) 29 | ) 30 | ) 31 | search_index.vector_search.profiles[0].vectorizer_name = "vectorizer" 32 | await index_client.create_or_update_index(search_index) 33 | print("Set up index") 34 | 35 | search_client = SearchClient(endpoint=os.environ["AZURE_SEARCH_ENDPOINT"], index_name=os.environ["AZURE_SEARCH_INDEX"], credential=credential) 36 | with open("sample-index/data.json") as f: 37 | documents = json.load(f) 38 | await search_client.upload_documents(documents) 39 | print("Set up data") 40 | 41 | await search_client.close() 42 | await index_client.close() 43 | await credential.close() 44 | 45 | if __name__ == "__main__": 46 | asyncio.run(main()) 47 | -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/media/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey Bonus - Agentic RAG/sample/media/demo.gif -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/models.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel, Field 2 | from typing import List, Optional, Dict, Any, Union, Literal, Annotated 3 | from copy import deepcopy 4 | from openai.types.chat import ChatCompletionMessageParam 5 | import chainlit as cl 6 | import re 7 | import os 8 | 9 | class SearchSources(BaseModel): 10 | """Retrieve sources from the Azure AI Search index""" 11 | search_query: str = Field(..., description="Query string to retrieve documents from azure search eg: 'Health care plan'") 12 | 13 | class AnswerReflectionWithStop(BaseModel): 14 | answer_reflection: str = Field( 15 | None, description="Explanation of why the previous answer was incorrect" 16 | ) 17 | follow_up_action: Literal["index_search", "web_search", "stop"] = Field( 18 | ..., description="Follow up action to take to try to fix the answer." 19 | ) 20 | follow_up_action_reason: str = Field( 21 | None, description="Explanation of why the follow up action was chosen" 22 | ) 23 | query: Optional[str] = Field( 24 | None, description="Optional new search query suggested by the reflection" 25 | ) 26 | 27 | class StopTool(BaseModel): 28 | """ 29 | There's no way to correct the previous answer. 30 | You only take this action if you've tried at least 1 action to correct the previous answer and it was unsuccessful. 31 | Take into account the previous actions taken to correct the previous answer. 32 | """ 33 | 34 | class WebSearchTool(BaseModel): 35 | """ 36 | The Chat History and Sources are missing key information that is needed to correct the previous answer 37 | You need to search the web for additional public information. 38 | Suggest a new web search query to find the missing information. 39 | If the answer was incorrect or incomplete because the required information was not in the text sources, and the required information is public knowledge, you should suggest searching the web for additional information. 40 | Hint: If we previously made an index_search for a similar topic, it is unlikely the internal index has this information. If it seems like it might be public knowledge, you should suggest searching the web. 41 | """ 42 | web_query: str = Field(..., description="Query string to retrieve documents from the web eg: 'Health care plan'") 43 | 44 | class IndexSearchTool(BaseModel): 45 | """ 46 | The Chat History and Sources are missing key information that is needed to correct the previous answer. 47 | You need to search an internal index for additional private information. 48 | Suggest a new search query to find the missing information. 49 | If the answer was incorrect or incomplete because the required information was not in the text sources, and the required information is internal to the organization, you should suggest searching the internal index for additional information. 50 | """ 51 | index_query: str = Field(..., description="Query string to retrieve documents from azure search eg: 'Health care plan'") 52 | 53 | class ToolChoiceReflection(BaseModel): 54 | answer_reflection: str = Field( 55 | ..., description="Explanation of why the previous answer was incorrect" 56 | ) 57 | follow_up_action: Union[StopTool, WebSearchTool, IndexSearchTool] = Field(..., description="Follow up action to take to try to fix the answer.") 58 | follow_up_action_reason: str = Field(..., description="Reason we took the follow up action") 59 | 60 | class EvaluatorResponse(BaseModel): 61 | type: str 62 | score: int 63 | explanation: str 64 | 65 | class Document(BaseModel): 66 | content: str 67 | id: str 68 | score: float 69 | title: str 70 | 71 | class ActionBase(BaseModel): 72 | type: str 73 | 74 | async def render(self): 75 | pass 76 | 77 | def step_name(self): 78 | return self.type 79 | 80 | def create_step(self): 81 | return True 82 | 83 | def render_during_step(self) -> bool: 84 | return True 85 | 86 | def get_input_props(self) -> Dict[str, Any]: 87 | return {} 88 | 89 | def get_output_props(self) -> Dict[str, Any]: 90 | return {} 91 | 92 | class QuestionAction(ActionBase): 93 | type: Literal["question"] = "question" 94 | question_text: str 95 | 96 | def create_step(self): 97 | return False 98 | 99 | async def render(self): 100 | await cl.Message(content=self.question_text, type="user_message").send() 101 | 102 | class QueryRewriteAction(ActionBase): 103 | type: Literal["query_rewrite"] = "query_rewrite" 104 | query: str 105 | rewritten_query: Optional[str] = None 106 | message_history: Optional[List[ChatCompletionMessageParam]] = None 107 | 108 | def get_input_props(self) -> Dict[str, Any]: 109 | return { "query": self.query, "message_history": self.message_history } 110 | 111 | def get_output_props(self) -> Dict[str, Any]: 112 | return { "rewritten_query": self.rewritten_query } 113 | 114 | class FilterDocumentsAction(ActionBase): 115 | type: Literal["filter_documents"] = "filter_documents" 116 | threshold: float 117 | documents: Optional[List[Document]] = None 118 | filtered_documents: List[Document] 119 | 120 | def get_input_props(self) -> Dict[str, Any]: 121 | return { "threshold": self.threshold, "documents": self.documents } 122 | 123 | def get_output_props(self) -> Dict[str, Any]: 124 | return { "filtered_documents": self.filtered_documents } 125 | 126 | class IndexSearchAction(ActionBase): 127 | type: Literal["index_search"] = "index_search" 128 | documents: List[Document] 129 | query: str 130 | rewritten_query: Optional[str] = None 131 | 132 | def get_input_props(self) -> Dict[str, Any]: 133 | return { "query": self.query, "rewritten_query": self.rewritten_query } 134 | 135 | def get_output_props(self) -> Dict[str, Any]: 136 | return { "documents": self.documents } 137 | 138 | class WebCitation(BaseModel): 139 | name: str 140 | url: str 141 | title: str 142 | 143 | class WebSearchAction(ActionBase): 144 | type: Literal["web_search"] = "web_search" 145 | query: str 146 | response_content: str 147 | citations: List[WebCitation] 148 | 149 | class AnswerSources(BaseModel): 150 | message_history: List[dict] # Assuming ChatCompletionMessageParam is a dict 151 | text_sources: Optional[List[Document]] = None 152 | public_sources: Optional[str] = None 153 | public_citations: Optional[List[WebCitation]] = None 154 | 155 | def create_context(self) -> str: 156 | context = "\n".join([message["role"] + ":\n" + message["content"] for message in self.message_history]) 157 | if self.text_sources: 158 | context = "Sources\n" 159 | content_no_newlines = lambda document: document.content.replace("\n", " ").replace("\r", " ") 160 | context += '\n '.join([f'{document.title}: {content_no_newlines(document)}' for document in self.text_sources]) 161 | if self.public_sources: 162 | context += "\nPublic Sources\n" 163 | public_sources = self.public_sources 164 | for details in self.public_citations: 165 | public_sources = public_sources.replace(details.name, f" [{details.title}]({details.url}) ") 166 | context += public_sources 167 | 168 | if len(context) == 0: 169 | context = "No sources found." 170 | return context 171 | 172 | def add_index_search(self, index_search_action: IndexSearchAction) -> "AnswerSources": 173 | text_sources = deepcopy(self.text_sources or []) 174 | for document in index_search_action.documents: 175 | if not any(existing_document.id == document.id for existing_document in text_sources): 176 | text_sources.append(document) 177 | return AnswerSources( 178 | message_history=deepcopy(self.message_history), 179 | text_sources=text_sources, 180 | public_sources=self.public_sources, 181 | public_citations=self.public_citations 182 | ) 183 | 184 | def add_web_search(self, web_search_action: WebSearchAction) -> "AnswerSources": 185 | public_citations = deepcopy(self.public_citations or []) 186 | for citation in web_search_action.citations: 187 | if not any(existing_citation.url == citation.url for existing_citation in public_citations): 188 | public_citations.append(citation) 189 | return AnswerSources( 190 | message_history=deepcopy(self.message_history), 191 | text_sources=self.text_sources, 192 | public_sources=(self.public_sources or "") + web_search_action.response_content, 193 | public_citations=public_citations 194 | ) 195 | 196 | @staticmethod 197 | def from_index_search(message_history: Optional[List[ChatCompletionMessageParam]], index_search_action: IndexSearchAction) -> "AnswerSources": 198 | return AnswerSources( 199 | message_history=deepcopy(message_history or []), 200 | text_sources=index_search_action.documents 201 | ) 202 | 203 | @staticmethod 204 | def from_web_search(message_history: Optional[List[ChatCompletionMessageParam]], web_search_action: WebSearchAction) -> "AnswerSources": 205 | return AnswerSources( 206 | message_history=deepcopy(message_history or []), 207 | public_sources=web_search_action.response_content, 208 | public_citations=web_search_action.citations 209 | ) 210 | 211 | class AnswerAction(ActionBase): 212 | type: Literal["answer"] = "answer" 213 | question_text: str 214 | answer_text: str 215 | answer_sources: AnswerSources 216 | 217 | async def render(self): 218 | await cl.Message(content=self.answer_text, elements=self.create_pdf_citations()).send() 219 | 220 | def step_name(self): 221 | return "generate_answer" 222 | 223 | def create_pdf_citations(self) -> List[cl.Pdf]: 224 | # Regex pattern to match citations like: 225 | # [AnyName.pdf#page=107] 226 | pattern = r'\[(?P[^\]]+\.pdf)#page=(?P\d+)\]' 227 | citation_names = set() 228 | citations = [] 229 | for match in re.finditer(pattern, self.answer_text): 230 | pdf_name = match.group('pdf') 231 | page = int(match.group('page')) 232 | pdf_path = os.path.join("data", pdf_name) 233 | if os.path.exists(pdf_path) and pdf_path not in citation_names: 234 | citations.append(cl.Pdf(name=pdf_name, path=pdf_path, page=page)) 235 | citation_names.add(pdf_path) 236 | 237 | return citations 238 | 239 | 240 | 241 | def render_during_step(self): 242 | return False 243 | 244 | def get_input_props(self) -> Dict[str, Any]: 245 | return { "question_text": self.question_text, "answer_sources": self.answer_sources.create_context() } 246 | 247 | def get_output_props(self) -> Dict[str, Any]: 248 | return { "answer_text": self.answer_text } 249 | 250 | def append_qa_message_history(self): 251 | self.answer_sources.message_history.append( 252 | { 253 | "role": "user", 254 | "content": self.question_text, 255 | } 256 | ) 257 | self.answer_sources.message_history.append( 258 | { 259 | "role": "assistant", 260 | "content": self.answer_text, 261 | } 262 | ) 263 | 264 | class EvaluationAction(ActionBase): 265 | type: Literal["evaluation"] = "evaluation" 266 | evaluations: List[EvaluatorResponse] 267 | answer: AnswerAction 268 | 269 | def step_name(self): 270 | return "evaluate_answer" 271 | 272 | async def render(self): 273 | await cl.Message(content="", elements=[ 274 | cl.CustomElement(name="EvaluationResults", props={"evaluations": [evaluation.model_dump() for evaluation in self.evaluations]}), 275 | ]).send() # Send the evaluation message to the user 276 | 277 | def get_input_props(self) -> Dict[str, Any]: 278 | return { "answer": self.answer } 279 | 280 | def get_output_props(self): 281 | return { "evaluations": self.evaluations } 282 | 283 | class ReflectionAction(ActionBase): 284 | type: Literal["reflection"] = "reflection" 285 | answer_text: str 286 | answer_reflection: str 287 | follow_up_action: Literal["index_search", "web_search", "stop"] 288 | follow_up_action_reason: str 289 | next_query: Optional[str] = None 290 | actions: List["Action"] 291 | 292 | def step_name(self): 293 | return "reflect_answer" 294 | 295 | async def render(self): 296 | await cl.Message( 297 | content="", 298 | elements=[ 299 | cl.CustomElement( 300 | name="ReflectionAction", 301 | props={ 302 | "answer_reflection": self.answer_reflection, 303 | "follow_up_action": self.follow_up_action, 304 | "follow_up_action_reason": self.follow_up_action_reason, 305 | "next_query": self.next_query 306 | } 307 | ) 308 | ]).send() # Send the reflection message to the user 309 | 310 | def get_input_props(self) -> Dict[str, Any]: 311 | return { "answer_text": self.answer_text } 312 | 313 | def get_output_props(self) -> Dict[str, Any]: 314 | return { "answer_reflection": self.answer_reflection, "follow_up_action": self.follow_up_action, "follow_up_action_reason": self.follow_up_action_reason, "next_query": self.next_query } 315 | 316 | class StopAction(ActionBase): 317 | type: Literal["stop"] = "stop" 318 | no_new_sources: bool = False 319 | 320 | Action = Annotated[Union[QuestionAction, QueryRewriteAction, FilterDocumentsAction, IndexSearchAction, WebSearchAction, AnswerAction, EvaluationAction, ReflectionAction, StopAction], Field(discriminator='type')] 321 | 322 | ReflectionAction.model_rebuild() 323 | 324 | class ActionList(BaseModel): 325 | actions: List[Action] 326 | 327 | def get_last_answer(self) -> Optional[AnswerAction]: 328 | last_answer_action = next( 329 | (action for action in reversed(self.actions) 330 | if isinstance(action, AnswerAction)), 331 | None 332 | ) 333 | return last_answer_action 334 | 335 | def get_last_query(self) -> Optional[str]: 336 | last_query_action = next( 337 | (action for action in reversed(self.actions) 338 | if isinstance(action, WebSearchAction) or isinstance(action, IndexSearchAction)), 339 | None 340 | ) 341 | if last_query_action: 342 | if isinstance(last_query_action, WebSearchAction): 343 | return last_query_action.query 344 | elif isinstance(last_query_action, IndexSearchAction): 345 | return last_query_action.rewritten_query or last_query_action.query 346 | 347 | return None 348 | 349 | def get_last_evaluation(self) -> Optional[EvaluationAction]: 350 | last_evaluation_action = next( 351 | (action for action in reversed(self.actions) 352 | if isinstance(action, EvaluationAction)), 353 | None 354 | ) 355 | return last_evaluation_action 356 | -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/prompts/chat_answer_question.prompty: -------------------------------------------------------------------------------- 1 | --- 2 | name: Chat 3 | description: Answer a question (with chat history) using solely text sources. 4 | model: 5 | api: chat 6 | sample: 7 | user_query: What does a product manager do that a CEO doesn't? 8 | past_messages: 9 | - role: user 10 | content: "What does a CEO do?" 11 | - role: assistant 12 | content: "A CEO, or Chief Executive Officer, is responsible for providing strategic direction and oversight to a company to ensure its long-term success and profitability. They develop and implement strategies and objectives for financial success and growth, provide guidance to the executive team, manage day-to-day operations, ensure compliance with laws and regulations, develop and maintain relationships with stakeholders, monitor industry trends, and represent the company in public events 12. [role_library.pdf#page=1][role_library.pdf#page=3]" 13 | text_sources: 14 | - "role_library.pdf#page=29: The Manager of Product Management will collaborate with internal teams, such as engineering, sales, marketing, and finance, as well as external partners, suppliers, and customers to ensure successful product execution. Responsibilities: · Lead the product management team and provide guidance on product strategy, design, development, and launch. · Develop and implement product life-cycle management processes. · Monitor and analyze industry trends to identify opportunities for new products. · Develop product marketing plans and go-to-market strategies. · Research customer needs and develop customer-centric product roadmaps. · Collaborate with internal teams to ensure product execution and successful launch. · Develop pricing strategies and cost models. · Oversee product portfolio and performance metrics. · Manage product development budget. · Analyze product performance and customer feedback to identify areas for improvement. Qualifications: · Bachelor's degree in business, engineering, or a related field. · At least 5 years of experience in product management. · Proven track record of successful product launches.\nrole_library.pdf#page=23: Company: Contoso Electronics Location: Anywhere Job Type: Full-Time Salary: Competitive, commensurate with experience Job Summary: The Senior Manager of Product Management will be responsible for leading the product management team at Contoso Electronics. This role includes developing strategies, plans and objectives for the product management team and managing the day-to-day operations. The Senior Manager of Product Management will be responsible for the successful launch of new products and the optimization of existing products. Responsibilities: · Develop and implement product management strategies, plans and objectives to maximize team performance. · Analyze competitive landscape and market trends to develop product strategies. · Lead the product management team in the development of product plans, roadmaps and launch plans. · Monitor the performance of product management team, analyze results and implement corrective action as needed. · Manage the product lifecycle, including product development, launch, and end of life. · Ensure product features and benefits meet customer requirements. · Establish and maintain relationships with key customers, partners, and vendors.\nrole_library.pdf#page=28: · 7+ years of experience in research and development in the electronics sector. · Proven track record of successfully designing, testing, and optimizing products. · Experience leading a team of researchers and engineers. · Excellent problem-solving and analytical skills. · Ability to work in a fast-paced environment and meet tight deadlines.· Knowledge of industry trends, technologies, and regulations. · Excellent communication and presentation skills. Manager of Product Management Job Title: Manager of Product Management, Contoso Electronics Job Summary: The Manager of Product Management is responsible for overseeing the product management team, driving product development and marketing strategy for Contoso Electronics. This individual will be accountable for the successful launch of new products and the implementation of product life-cycle management processes. The Manager of Product Management will collaborate with internal teams, such as engineering, sales, marketing, and finance, as well as external partners, suppliers, and customers to ensure successful product execution." 15 | --- 16 | system: 17 | Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers. 18 | Answer ONLY with the facts listed in the list of public and internal sources below. 19 | If there isn't enough information below, say you don't know. 20 | Do not generate answers that don't use the public or internal sources below. 21 | If asking a clarifying question to the user would help, ask the question. 22 | If the question is not in English, answer in the language used in the question. 23 | 24 | Internal sources are under Sources and are from an internal source 25 | Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]. 26 | Public sources are from a web search and may not be fully accurate or up-to-date. Always prioritize internal sources for accuracy. 27 | Public sources have their own citation format. 28 | There is a single summary of the search results 29 | Citations are provided inside this summary 30 | Citation format is a markdown URL link, like this: [source_name](https://example.com/source_name.pdf#page=1). 31 | Cite all markdown links if you reference the summary at all 32 | When using public sources, make sure to mention that they are from a web search and may not be fully accurate or up-to-date. 33 | 34 | {% if previous_answer %} 35 | 36 | Assistant has previously answered this question incorrectly. 37 | Use the provided information below to correct the previous answer. 38 | When correcting the answer, only use information in the list of sources below. 39 | Be concise and clear in your explanation. 40 | Be sure to address the concerns in the provided reflection and evaluations. 41 | 42 | Previous Answer: 43 | {{ previous_answer }} 44 | 45 | Reflection on Previous Answer: 46 | {{ previous_answer_reflection }} 47 | 48 | Evaluations of Previous Answer: 49 | {{ previous_answer_evaluations }} 50 | 51 | {% endif %} 52 | 53 | user: 54 | {{ user_query }} 55 | 56 | {{ context }} -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/prompts/chat_query_rewrite.prompty: -------------------------------------------------------------------------------- 1 | --- 2 | name: Rewrite RAG query 3 | description: Suggest the optimal search query based on the user's query, examples, and chat history. 4 | model: 5 | api: chat 6 | sample: 7 | user_query: Does it include hearing? 8 | past_messages: 9 | - role: user 10 | content: "What is included in my Northwind Health Plus plan that is not in standard?" 11 | - role: assistant 12 | content: "The Northwind Health Plus plan includes coverage for emergency services, mental health and substance abuse coverage, and out-of-network services, which are not included in the Northwind Standard plan. [Benefit_Options.pdf#page=3]" 13 | --- 14 | system: 15 | Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base. 16 | You have access to Azure AI Search index with 100's of documents. 17 | Generate a search query based on the conversation and the new question. 18 | Do not include cited source filenames and document names e.g. info.txt or doc.pdf in the search query terms. 19 | Do not include any text inside [] or <<>> in the search query terms. 20 | Do not include any special characters like '+'. 21 | If the question is not in English, translate the question to English before generating the search query. 22 | If you cannot generate a search query, return just the number 0. 23 | 24 | user: 25 | How did crypto do last year? 26 | 27 | assistant: 28 | Summarize Cryptocurrency Market Dynamics from last year 29 | 30 | user: 31 | What are my health plans? 32 | 33 | assistant: 34 | Show available health plans 35 | 36 | {% for message in past_messages %} 37 | {{ message["role"] }}: 38 | {{ message["content"] }} 39 | {% endfor %} 40 | 41 | user: 42 | Generate search query for: {{ user_query }} -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/prompts/chat_reflect_answer.prompty: -------------------------------------------------------------------------------- 1 | --- 2 | name: Chat 3 | description: Determine why a previous answer to a question (with chat history) was wrong using solely text sources. 4 | model: 5 | api: chat 6 | sample: 7 | user_query: Who ensures the compliance of the Northwind Health Plus plan with state regulations? 8 | ungrounded_answer: "The compliance of the Northwind Health Plus plan with state regulations is ensured by Northwind Health, which strives to follow all applicable laws, including ERISA, the ACA, and the MHPAEA. The plan is required to meet minimum standards set by these laws and provide necessary benefits to participants [Northwind_Health_Plus_Benefits_Details.pdf#page=90][Northwind_Standard_Benefits_Details.pdf#page=85]." 9 | past_messages: 10 | - role: user 11 | content: "What does a CEO do?" 12 | - role: assistant 13 | content: "A CEO, or Chief Executive Officer, is responsible for providing strategic direction and oversight to a company to ensure its long-term success and profitability. They develop and implement strategies and objectives for financial success and growth, provide guidance to the executive team, manage day-to-day operations, ensure compliance with laws and regulations, develop and maintain relationships with stakeholders, monitor industry trends, and represent the company in public events 12. [role_library.pdf#page=1][role_library.pdf#page=3]" 14 | text_sources: 15 | - "Northwind_Health_Plus_Benefits_Details.pdf#page=90: However, some cases may take longer to resolve. Finally, be sure to keep a record of all correspondence with Northwind Health Plus. This includes any phone calls, emails, or letters you send or receive. This will help ensure that your complaint or appeal is being addressed in a timely manner. OTHER INFORMATION ABOUT THIS PLAN Conformity With The Law OTHER INFORMATION ABOUT THIS PLAN - CONFORMITY WITH THE LAW Northwind Health Plus is in compliance with applicable state and federal laws and regulations, including the Employee Retirement Income Security Act (ERISA). This plan is also compliant with the Affordable Care Act (ACA) and the Mental Health Parity and Addiction Equity Act (MHPAEA). Under ERISA, the plan must provide certain benefits, such as protection from discrimination, vesting rights, and reasonable notice of plan changes. The plan must also provide detailed information to participants, such as a Summary Plan Description (SPD), which outlines plan provisions and benefits. Under the ACA, Northwind Health Plus must provide essential health benefits, such as ambulatory patient services, hospitalization, \n If you have exhausted the appeals process within Northwind Health and you believe that your appeal or grievance was not adequately addressed, you can contact your state's insurance department or the federal Department of Health and Human Services for more information about the external review process. OTHER INFORMATION ABOUT THIS PLAN Conformity With The Law OTHER INFORMATION ABOUT THIS PLAN: Conformity With The LawContoso and Northwind Health understand the importance of following the law when it comes to employee benefits and strive to ensure that the Northwind Standard plan is in compliance with all applicable laws. ERISA Requirements The Employee Retirement Income Security Act of 1974 (ERISA) sets minimum standards for pension and health plans in private industry. Northwind Standard meets the requirements established by ERISA, including providing employees with a Summary Plan Description and an annual report. COBRA The Consolidated Omnibus Budget Reconciliation Act (COBRA) gives employees and their families the right to continue their health coverage after a job loss or other qualifying event.\n Northwind_Health_Plus_Benefits_Details.pdf" 16 | --- 17 | system: 18 | Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers. 19 | Assistant is reflecting on why a previous answer was wrong. Assistant needs to determine how to correct the previous answer. 20 | Assistant has access to an internal search index to find additional information about healthcare plan and employee handbook questions 21 | Assistant has access to the web to find additional public information about healthcare 22 | 23 | If the question is not in English, answer in the language used in the question. 24 | If referencing a source, use square brackets to reference the source, for example [info1.txt]. 25 | Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]. 26 | 27 | Explain *why* the previous answer was incorrect and *why* the follow up action is needed to correct the previous answer. 28 | You should *only* do a new internal search if the next query is significantly different than previous queries 29 | If the next query is not significantly different than previous queries, you should not do a new internal search. 30 | 31 | user: 32 | 33 | Previous Query: 34 | {{ previous_query }} 35 | 36 | Previous Answer: 37 | {{ previous_answer }} 38 | 39 | Previous Answer Evaluation: 40 | {{ previous_answer_evaluation }} 41 | 42 | Previous Sources: 43 | {{ previous_sources }} 44 | 45 | -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/public/elements/EvaluationResults.jsx: -------------------------------------------------------------------------------- 1 | import { Card, CardHeader, CardTitle, CardContent } from "@/components/ui/card" 2 | 3 | export default function EvaluationResults() { 4 | const relevance = props.evaluations.find(e => e.type === 'relevance'); 5 | const groundedness = props.evaluations.find(e => e.type === 'groundedness'); 6 | 7 | // Utility to convert a score (1 to 5) into a percentage (width for bar) 8 | const progressWidth = score => `${(score / 5) * 100}%` 9 | return ( 10 | 11 | 12 | Evaluation Results 13 | 14 | 15 |
16 |
Groundedness
17 |
18 |
22 |
23 |
24 | {groundedness.score} out of 5 25 |
26 |
27 | {groundedness.explanation} 28 |
29 |
30 |
31 |
Relevance
32 |
33 |
37 |
38 |
39 | {relevance.score} out of 5 40 |
41 |
42 | {relevance.explanation} 43 |
44 |
45 | 46 | 47 | ) 48 | } -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/public/elements/ReflectionAction.jsx: -------------------------------------------------------------------------------- 1 | import { Card, CardHeader, CardTitle, CardContent } from "@/components/ui/card" 2 | import { StopCircle, Globe, Pencil } from "lucide-react" 3 | 4 | export default function CreateElement() { 5 | titles = { 6 | "stop": { 7 | "title": "🛑", 8 | "description": "Stop" 9 | }, 10 | "web_search": { 11 | "title": "🌐", 12 | "description": "Web Search" 13 | }, 14 | "index_search": { 15 | "title": "📝", 16 | "description": "Index Search" 17 | } 18 | } 19 | return ( 20 | 21 | 22 | Next Action: {titles[props.follow_up_action]['title']} {titles[props.follow_up_action]['description']} 23 | 24 | 25 |
26 | Reason: {props.follow_up_action_reason} 27 |
28 |
29 | Answer Reflection: {props.answer_reflection} 30 |
31 | {props.next_query && 32 |
33 | Next Query: {props.next_query} 34 |
35 | } 36 | 37 | 38 | ) 39 | } -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey Bonus - Agentic RAG/sample/public/favicon.ico -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/public/logo_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey Bonus - Agentic RAG/sample/public/logo_dark.png -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/public/logo_light.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/Journey Bonus - Agentic RAG/sample/public/logo_light.png -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/requirements.txt: -------------------------------------------------------------------------------- 1 | azure-search-documents==11.5.2 2 | azure-ai-evaluation 3 | azure-ai-projects 4 | azure-identity 5 | aiohttp 6 | openai 7 | chainlit 8 | python-dotenv 9 | prompty -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/sample-index/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sample", 3 | "fields": [ 4 | { 5 | "name": "id", 6 | "type": "Edm.String", 7 | "searchable": false, 8 | "filterable": false, 9 | "retrievable": true, 10 | "stored": true, 11 | "sortable": false, 12 | "facetable": false, 13 | "key": true, 14 | "synonymMaps": [] 15 | }, 16 | { 17 | "name": "content", 18 | "type": "Edm.String", 19 | "searchable": true, 20 | "filterable": false, 21 | "retrievable": true, 22 | "stored": true, 23 | "sortable": false, 24 | "facetable": false, 25 | "key": false, 26 | "analyzer": "en.microsoft", 27 | "synonymMaps": [] 28 | }, 29 | { 30 | "name": "embedding", 31 | "type": "Collection(Edm.Single)", 32 | "searchable": true, 33 | "filterable": false, 34 | "retrievable": true, 35 | "stored": true, 36 | "sortable": false, 37 | "facetable": false, 38 | "key": false, 39 | "dimensions": 3072, 40 | "vectorSearchProfile": "embedding_config", 41 | "synonymMaps": [] 42 | }, 43 | { 44 | "name": "sourcepage", 45 | "type": "Edm.String", 46 | "searchable": false, 47 | "filterable": true, 48 | "retrievable": true, 49 | "stored": true, 50 | "sortable": false, 51 | "facetable": true, 52 | "key": false, 53 | "synonymMaps": [] 54 | } 55 | ], 56 | "scoringProfiles": [], 57 | "suggesters": [], 58 | "analyzers": [], 59 | "normalizers": [], 60 | "tokenizers": [], 61 | "tokenFilters": [], 62 | "charFilters": [], 63 | "similarity": { 64 | "@odata.type": "#Microsoft.Azure.Search.BM25Similarity" 65 | }, 66 | "semantic": { 67 | "configurations": [ 68 | { 69 | "name": "default", 70 | "flightingOptIn": false, 71 | "prioritizedFields": { 72 | "prioritizedContentFields": [ 73 | { 74 | "fieldName": "content" 75 | } 76 | ], 77 | "prioritizedKeywordsFields": [] 78 | } 79 | } 80 | ] 81 | }, 82 | "vectorSearch": { 83 | "algorithms": [ 84 | { 85 | "name": "hnsw_config", 86 | "kind": "hnsw", 87 | "hnswParameters": { 88 | "metric": "cosine", 89 | "m": 4, 90 | "efConstruction": 400, 91 | "efSearch": 500 92 | } 93 | } 94 | ], 95 | "profiles": [ 96 | { 97 | "name": "embedding_config", 98 | "algorithm": "hnsw_config" 99 | } 100 | ], 101 | "vectorizers": [], 102 | "compressions": [] 103 | } 104 | } -------------------------------------------------------------------------------- /Journey Bonus - Agentic RAG/sample/sample.env: -------------------------------------------------------------------------------- 1 | AZURE_OPENAI_ENDPOINT=https://your-service.openai.azure.com 2 | AZURE_OPENAI_API_VERSION=2025-03-01-preview 3 | AZURE_OPENAI_API_KEY=your-openai-key 4 | AZURE_SEARCH_ENDPOINT=https://your-service.search.windows.net 5 | AZURE_SEARCH_INDEX=agentic-rag-sample 6 | AZURE_SEARCH_KEY= 7 | AI_PROJECT_CONN_STR="your-project-connection-string" 8 | BING_GROUNDING_NAME=your-bing-grounding-connection-name 9 | AZURE_OPENAI_EMBEDDING_DEPLOYMENT=text-embedding-3-large 10 | AZURE_OPENAI_GENERATOR_DEPLOYMENT=gpt-4o-mini 11 | AZURE_OPENAI_REWRITER_DEPLOYMENT=gpt-4o-mini 12 | AZURE_OPENAI_EVALUATOR_DEPLOYMENT=gpt-4o 13 | AZURE_OPENAI_REFLECTION_DEPLOYMENT=gpt-4o 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RAG Time: Ultimate Guide to Mastering RAG 2 | 3 | ![RAG Time Banner](./images/agenda-content.png) 4 | 5 | 6 | [![Calendar blockers](https://img.shields.io/badge/%F0%9F%93%86-add%20to%20calendar-blue?style=for-the-badge)](https://aka.ms/rag-time/calendar) 7 | 8 | 9 | [![GitHub watchers](https://img.shields.io/github/watchers/microsoft/rag-time.svg?style=social&label=Watch)](https://GitHub.com/microsoft/rag-time/watchers) 10 | [![GitHub forks](https://img.shields.io/github/forks/microsoft/rag-time.svg?style=social&label=Fork)](https://github.com/microsoft/rag-time/fork) 11 | [![GitHub stars](https://img.shields.io/github/stars/microsoft/rag-time?style=social&label=Star)](https://GitHub.com/microsoft/rag-time/stargazers) 12 | 13 | [![Azure AI Community Discord](https://dcbadge.vercel.app/api/server/ByRwuEEgH4)](https://discord.gg/REmjGvvFpW) 14 | 15 | 🚀 Master RAG with RAG Time! Learn how to build smarter AI applications with Retrieval-Augmented Generation. This repo includes step-by-step guides, live coding samples, and expert insights—everything you need to go from beginner to RAG pro! 16 | 17 | **📺 All episodes of RAG Time are live! [Watch the full series now on YouTube.](https://aka.ms/rag-time/journeys)** 18 | 19 | ## 🤔 Why Learn RAG with RAG Time? 20 | 21 | - **Expert-Led** – Learn directly from AI & Microsoft experts. 22 | - **Future-Proof Your AI Skills** – Stay ahead with cutting-edge retrieval techniques. 23 | - **Exclusive Video Content** – Interviews with AI leaders, in-depth tech talks, and practical demonstrations. 24 | - **Sample Code** – Follow along with ready-to-use implementations and best practices. 25 | - **Doodles & Visual Explanations** – Complex concepts made simple with engaging visuals. 26 | 27 | ## 🚀 Get Started in 3 Easy Steps 28 | 29 | 1. [Fork](https://github.com/microsoft/rag-time/fork) the repository. 30 | 2. Clone the repository to your local machine: 31 | ```bash 32 | git clone https://github.com/your-org/rag-time.git 33 | cd rag-time 34 | ``` 35 | 3. Navigate to the Journey of your choice and follow the README Instructions. 36 | 37 | ## Learning Journeys 38 | RAG Time runs every **Wednesday at 9AM PT** from **March 5th to April 2nd**. Each journey covers unique topics with leadership insights, tech talks, and code samples 39 | | **Journey Page** | **Description** | **Video** | **Code Sample** | **Blog** | 40 | |------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------|-------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------| 41 | | [RAG and Knowledge Retrieval Fundamentals](./Journey%201%20-%20RAG%20and%20Knowledge%20Retrieval%20Fundamentals/README.md) | Understand the strategic importance of RAG and knowledge retrieval | [Watch now](https://aka.ms/rag-time/journey1) | [Sample](./Journey%201%20-%20RAG%20and%20Knowledge%20Retrieval%20Fundamentals/sample/1-RAG-Fundamentals.ipynb) | [Journey 1](https://aka.ms/rag-time/journey1-blog) | 42 | | [Build the Ultimate Retrieval System](./Journey%202%20-%20Build%20the%20Ultimate%20Retrieval%20System%20for%20RAG/README.md) | Explore how Azure AI Search powers retrieval system | [Watch now](https://aka.ms/rag-time/journey2) | [Sample](./Journey%202%20-%20Build%20the%20Ultimate%20Retrieval%20System%20for%20RAG/sample/2-Build-The-Ultimate-Retrieval-System-For-RAG.ipynb) | [Journey 2](https://aka.ms/rag-time/journey2-blog) | 43 | | [Optimize Your Vector Index at Scale](./Journey%203%20-%20Optimize%20your%20Vector%20Index%20for%20Scale/README.md) | Learn real-world optimization techniques for scaling vector indexes | [Watch now](https://aka.ms/rag-time/journey3) | [Sample](./Journey%203%20-%20Optimize%20your%20Vector%20Index%20for%20Scale/sample/3-Vector-Compression.ipynb) | [Journey 3](https://aka.ms/rag-time/journey3-blog) | 44 | | [RAG for All Your Data](./Journey%204%20-%20RAG%20for%20All%20your%20Data%20Multimodal%20and%20Beyond/README.md) | Discover how multimodal data can be indexed and retrieved | [Watch now](https://aka.ms/rag-time/journey4) | [Sample](./Journey%204%20-%20RAG%20for%20All%20your%20Data%20Multimodal%20and%20Beyond/sample/README.md) | [Journey 4](https://aka.ms/rag-time/journey4-blog) | 45 | | [Hero Use-Cases for RAG](./Journey%205%20-%20Hero%20use%20cases%20for%20RAG/README.md) | Get inspired by hero use cases of RAG in action | [Watch now](https://aka.ms/rag-time/journey5) | [Sample](./Journey%205%20-%20Hero%20use%20cases%20for%20RAG/sample/README.md) | [Journey 5](https://aka.ms/rag-time/journey5-blog) | 46 | | [Agentic RAG](./Journey%20Bonus%20-%20Agentic%20RAG/README.md) | Learn about Agentic RAG an build your first Agentic RAG solution | [Ask your questions on Discord!](https://aka.ms/rag-time/discord) | [Sample](./Journey%20Bonus%20-%20Agentic%20RAG/sample/README.md) | [Bonus Journey](http://aka.ms/rag-time/bonusjourney-blog) | 47 | 48 | ### Content Release Schedule 49 | 50 | ![journeys weekly](./images/journeys-weekly.png) 51 | 52 | ## 🙏 Get Involved 53 | 54 | We'd love to see you contributing to our repo and engaging with the experts with your questions! 55 | 56 | - 🤔 Do you have suggestions or have you found spelling or code errors? [Raise an issue](https://github.com/microsoft/rag-time/issues) or [Create a pull request](https://github.com/microsoft/rag-time/pulls). 57 | - 🚀 If you get stuck or have any questions about RAG, join our [Azure AI Community Discord](https://discord.com/channels/1113626258182504448/1343540943533637663). 58 | 59 | ## Meet the RAG Time Speakers 60 | 61 | ![The Experts Behind RAG Time](./images/speakers.png) 62 | 63 | ## Contributing 64 | 65 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 66 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 67 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 68 | 69 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 70 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 71 | provided by the bot. You will only need to do this once across all repos using our CLA. 72 | 73 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 74 | For more information, see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 75 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 76 | 77 | ## Trademarks 78 | 79 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 80 | trademarks or logos is subject to and must follow 81 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 82 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 83 | Any use of third-party trademarks or logos is subject to those third parties' policies. 84 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. 7 | - **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /images/agenda-content.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/agenda-content.png -------------------------------------------------------------------------------- /images/banner-rag-time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/banner-rag-time.png -------------------------------------------------------------------------------- /images/j1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/j1.png -------------------------------------------------------------------------------- /images/j2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/j2.png -------------------------------------------------------------------------------- /images/j3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/j3.png -------------------------------------------------------------------------------- /images/j4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/j4.png -------------------------------------------------------------------------------- /images/j5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/j5.png -------------------------------------------------------------------------------- /images/journeys-weekly.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/journeys-weekly.png -------------------------------------------------------------------------------- /images/repo-banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/repo-banner.png -------------------------------------------------------------------------------- /images/speakers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/speakers.png -------------------------------------------------------------------------------- /images/visuals/J1-recap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/visuals/J1-recap.png -------------------------------------------------------------------------------- /images/visuals/J2-recap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/visuals/J2-recap.png -------------------------------------------------------------------------------- /images/visuals/J3-recap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/visuals/J3-recap.png -------------------------------------------------------------------------------- /images/visuals/J4-recap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/visuals/J4-recap.png -------------------------------------------------------------------------------- /images/visuals/J5-recap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/rag-time/b9dec9c52c1e2dc2da786c40dab632527e52dd61/images/visuals/J5-recap.png --------------------------------------------------------------------------------