├── .DS_Store ├── .github └── workflows │ └── jekyll-gh-pages.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── Gemfile ├── Gemfile.lock ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── _config.yml ├── docs ├── .DS_Store ├── 01_intro_to_llms │ ├── .DS_Store │ ├── 01_01.md │ ├── 01_02.md │ ├── 01_03.md │ ├── 01_intro_to_llms.md │ ├── 01_setup.md │ └── images │ │ ├── bootstrapping_env_vars.png │ │ ├── bootstrapping_environments.png │ │ ├── content_safety01.png │ │ ├── content_safety02.png │ │ ├── content_safety_results.png │ │ ├── deploy_model.png │ │ ├── deploy_model_created.png │ │ ├── deploy_model_phi3.png │ │ ├── deploy_model_serverless.png │ │ ├── deploy_model_serverless_name.png │ │ ├── enable_github_actions.png │ │ ├── labgrab13.png │ │ ├── labgrab14.png │ │ ├── labgrab15.png │ │ ├── labgrab16.png │ │ ├── phi35_chat_message.png │ │ ├── phi35_response.png │ │ └── phi35_system_message.png ├── 02_build_llm_flows │ ├── .DS_Store │ ├── 02_01.md │ ├── 02_build_llm_flows.md │ ├── files │ │ ├── updated_chat.prompty │ │ └── updated_chat_request.py │ └── images │ │ ├── build_aistudio_code.png │ │ ├── build_authenticate.png │ │ ├── build_breakpoint.png │ │ ├── build_clone.png │ │ ├── build_compute.png │ │ ├── build_container.png │ │ ├── build_debug_variables.png │ │ ├── build_flow.png │ │ ├── build_launch_vscode.png │ │ ├── build_run_breakpoint.png │ │ ├── build_run_debug.png │ │ ├── build_run_flow.png │ │ ├── build_terminal.png │ │ ├── deployment02.png │ │ ├── trace00.png │ │ ├── trace01.png │ │ └── trace02.png ├── 03_evaluate_and_deploy_llms │ ├── .DS_Store │ ├── 03_01.md │ ├── 03_02.md │ ├── 03_evaluate_and_deploy_llms.md │ └── images │ │ ├── deploy-script-additions.png │ │ ├── deployment01.png │ │ ├── deployment02.png │ │ ├── deployment03.png │ │ ├── deployment04.png │ │ ├── deployment05.png │ │ ├── deployment06.png │ │ ├── env-additions.png │ │ ├── evaluate_adversarial01.png │ │ ├── evaluate_adversarial02.png │ │ ├── evaluate_prompty.png │ │ ├── evaluate_qa01.png │ │ ├── evaluate_qa02.png │ │ ├── safety-eval-error.png │ │ └── testdeploy.png ├── 04_monitoring_llm_flows │ ├── .DS_Store │ ├── 04_01.md │ ├── 04_monitoring_llms.md │ ├── draft backup │ │ └── Exercise04.md │ └── images │ │ ├── dash_endpoint.png │ │ ├── dash_insights.png │ │ ├── dash_operational.png │ │ ├── dash_quality.png │ │ ├── lab4grab1.png │ │ ├── lab4grab10.png │ │ ├── lab4grab11.png │ │ ├── lab4grab12.png │ │ ├── lab4grab13.png │ │ ├── lab4grab14.png │ │ ├── lab4grab15.png │ │ ├── lab4grab16.png │ │ ├── lab4grab17.png │ │ ├── lab4grab18.png │ │ ├── lab4grab19.png │ │ ├── lab4grab2.png │ │ ├── lab4grab20.png │ │ ├── lab4grab21.png │ │ ├── lab4grab22.png │ │ ├── lab4grab23.png │ │ ├── lab4grab24.png │ │ ├── lab4grab3.png │ │ ├── lab4grab4.png │ │ ├── lab4grab5.png │ │ ├── lab4grab6.png │ │ ├── lab4grab7.png │ │ ├── lab4grab8.png │ │ ├── lab4grab9.png │ │ ├── monitor01.png │ │ ├── monitor02.png │ │ ├── monitor03.png │ │ └── monitor04.png └── 05_automating_everything │ ├── 05_01.md │ ├── 05_automating_everything.md │ └── images │ ├── git_workflow_branching.png │ ├── git_workflow_pipelines.png │ ├── lab6grab1.png │ ├── lab6grab10.png │ ├── lab6grab11.png │ ├── lab6grab12.png │ ├── lab6grab13.png │ ├── lab6grab14.png │ ├── lab6grab15.png │ ├── lab6grab16.png │ ├── lab6grab17.png │ ├── lab6grab18.png │ ├── lab6grab2.png │ ├── lab6grab3.png │ ├── lab6grab4.png │ ├── lab6grab4b.png │ ├── lab6grab4c.png │ ├── lab6grab5.png │ ├── lab6grab6.png │ ├── lab6grab7.png │ ├── lab6grab8.png │ ├── lab6grab9.png │ ├── large-language-model-operations-prompt-flow-process.png │ └── llmops-with-prompt-flow-diagram.png ├── index.md └── resource-provider-connected-resources.svg /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/.DS_Store -------------------------------------------------------------------------------- /.github/workflows/jekyll-gh-pages.yml: -------------------------------------------------------------------------------- 1 | # Sample workflow for building and deploying a Jekyll site to GitHub Pages 2 | name: Deploy Jekyll with GitHub Pages dependencies preinstalled 3 | 4 | on: 5 | # Runs on pushes targeting the default branch 6 | push: 7 | branches: ["main"] 8 | 9 | # Allows you to run this workflow manually from the Actions tab 10 | workflow_dispatch: 11 | 12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 13 | permissions: 14 | contents: read 15 | pages: write 16 | id-token: write 17 | 18 | # Allow only one concurrent deployment to GitHub Pages 19 | concurrency: 20 | group: "pages" 21 | cancel-in-progress: true 22 | 23 | jobs: 24 | # Build job 25 | build: 26 | runs-on: ubuntu-latest 27 | steps: 28 | - name: Checkout 29 | uses: actions/checkout@v4 30 | - name: Setup Ruby 31 | uses: ruby/setup-ruby@v1 32 | with: 33 | ruby-version: '3.1' # Not needed with a .ruby-version file 34 | bundler-cache: true # runs 'bundle install' and caches installed gems automatically 35 | cache-version: 0 # Increment this number if you need to re-download cached gems 36 | - name: Setup Pages 37 | id: pages 38 | uses: actions/configure-pages@v4 39 | - name: Build with Jekyll 40 | # Outputs to the './_site' directory by default 41 | run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}" 42 | env: 43 | JEKYLL_ENV: production 44 | - name: Upload artifact 45 | uses: actions/upload-pages-artifact@v3 46 | 47 | # Deployment job 48 | deploy: 49 | environment: 50 | name: github-pages 51 | url: ${{ steps.deployment.outputs.page_url }} 52 | runs-on: ubuntu-latest 53 | needs: build 54 | steps: 55 | - name: Deploy to GitHub Pages 56 | id: deployment 57 | uses: actions/deploy-pages@v4 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | docs/.DS_Store 3 | .DS_Store 4 | _local/ -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem "jekyll", "~> 4.3" # installed by `gem jekyll` 4 | # gem "webrick" # required when using Ruby >= 3 and Jekyll <= 4.2.2 5 | 6 | gem "just-the-docs", "0.4.2" # pinned to the current release 7 | # gem "just-the-docs" # always download the latest release 8 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | addressable (2.8.1) 5 | public_suffix (>= 2.0.2, < 6.0) 6 | colorator (1.1.0) 7 | concurrent-ruby (1.1.10) 8 | em-websocket (0.5.3) 9 | eventmachine (>= 0.12.9) 10 | http_parser.rb (~> 0) 11 | eventmachine (1.2.7) 12 | ffi (1.15.5) 13 | forwardable-extended (2.6.0) 14 | http_parser.rb (0.8.0) 15 | i18n (1.12.0) 16 | concurrent-ruby (~> 1.0) 17 | jekyll (4.3.0) 18 | addressable (~> 2.4) 19 | colorator (~> 1.0) 20 | em-websocket (~> 0.5) 21 | i18n (~> 1.0) 22 | jekyll-sass-converter (>= 2.0, < 4.0) 23 | jekyll-watch (~> 2.0) 24 | kramdown (~> 2.3, >= 2.3.1) 25 | kramdown-parser-gfm (~> 1.0) 26 | liquid (~> 4.0) 27 | mercenary (>= 0.3.6, < 0.5) 28 | pathutil (~> 0.9) 29 | rouge (>= 3.0, < 5.0) 30 | safe_yaml (~> 1.0) 31 | terminal-table (>= 1.8, < 4.0) 32 | webrick (~> 1.7) 33 | jekyll-sass-converter (2.2.0) 34 | sassc (> 2.0.1, < 3.0) 35 | jekyll-seo-tag (2.8.0) 36 | jekyll (>= 3.8, < 5.0) 37 | jekyll-watch (2.2.1) 38 | listen (~> 3.0) 39 | just-the-docs (0.4.2) 40 | jekyll (>= 3.8.5) 41 | jekyll-seo-tag (>= 2.0) 42 | rake (>= 12.3.1) 43 | kramdown (2.4.0) 44 | rexml 45 | kramdown-parser-gfm (1.1.0) 46 | kramdown (~> 2.0) 47 | liquid (4.0.3) 48 | listen (3.7.1) 49 | rb-fsevent (~> 0.10, >= 0.10.3) 50 | rb-inotify (~> 0.9, >= 0.9.10) 51 | mercenary (0.4.0) 52 | pathutil (0.16.2) 53 | forwardable-extended (~> 2.6) 54 | public_suffix (5.0.0) 55 | rake (13.0.6) 56 | rb-fsevent (0.11.2) 57 | rb-inotify (0.10.1) 58 | ffi (~> 1.0) 59 | rexml (3.3.9) 60 | rouge (4.0.0) 61 | safe_yaml (1.0.5) 62 | sassc (2.4.0) 63 | ffi (~> 1.9) 64 | terminal-table (3.0.2) 65 | unicode-display_width (>= 1.1.1, < 3) 66 | unicode-display_width (2.3.0) 67 | webrick (1.7.0) 68 | 69 | PLATFORMS 70 | arm64-darwin-21 71 | x86_64-darwin-19 72 | x86_64-linux 73 | 74 | DEPENDENCIES 75 | jekyll (~> 4.3) 76 | just-the-docs (= 0.4.2) 77 | 78 | BUNDLED WITH 79 | 2.3.9 80 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Project 2 | 3 | > This repo has been populated by an initial template to help get you started. Please 4 | > make sure to update the content to build a great experience for community-building. 5 | 6 | As the maintainer of this project, please make a few updates: 7 | 8 | - Improving this README.MD file to provide a great experience 9 | - Updating SUPPORT.MD with content about this project's support experience 10 | - Understanding the security reporting process in SECURITY.MD 11 | - Remove this section from the README 12 | 13 | ## Contributing 14 | 15 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 16 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 17 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 18 | 19 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 20 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 21 | provided by the bot. You will only need to do this once across all repos using our CLA. 22 | 23 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 24 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 25 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 26 | 27 | ## Trademarks 28 | 29 | This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 30 | trademarks or logos is subject to and must follow 31 | [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). 32 | Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. 33 | Any use of third-party trademarks or logos are subject to those third-party's policies. 34 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # TODO: The maintainer of this repo has not yet edited this file 2 | 3 | **REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? 4 | 5 | - **No CSS support:** Fill out this template with information about how to file issues and get help. 6 | - **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. 7 | - **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. 8 | 9 | *Then remove this first heading from this SUPPORT.MD file before publishing your repo.* 10 | 11 | # Support 12 | 13 | ## How to file issues and get help 14 | 15 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 16 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 17 | feature request as a new Issue. 18 | 19 | For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE 20 | FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER 21 | CHANNEL. WHERE WILL YOU HELP PEOPLE?**. 22 | 23 | ## Microsoft Support Policy 24 | 25 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 26 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | title: Operationalize LLMs with GenAIOps automation 2 | description: This lab teaches you how to create and test LLMs inside Azure AI Studio 3 | theme: just-the-docs 4 | 5 | # url: https://microsoft.github.io/TechExcel-Operationalize-LLMs-with-LLMOps-automation// 6 | 7 | aux_links: 8 | "TechExcel: Operationalize LLMs with LLMOps automation": 9 | - "https://github.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation" 10 | callouts_level: quiet # or loud 11 | callouts: 12 | highlight: 13 | color: yellow 14 | important: 15 | title: Important 16 | color: blue 17 | new: 18 | title: New 19 | color: green 20 | note: 21 | title: Note 22 | color: purple 23 | warning: 24 | title: Warning 25 | color: red 26 | -------------------------------------------------------------------------------- /docs/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/.DS_Store -------------------------------------------------------------------------------- /docs/01_intro_to_llms/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/.DS_Store -------------------------------------------------------------------------------- /docs/01_intro_to_llms/01_01.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '01: Use the AzureAI Foundry Playground' 3 | layout: default 4 | nav_order: 2 5 | parent: 'Exercise 01: Introduction to LLMs and Azure AI Services' 6 | --- 7 | 8 | # Task 01: Use the Azure AI Foundry Playground 9 | 10 | ## Introduction 11 | 12 | In an effort to evaluate the use of AI in their organization, Lamna Healthcare has identified the use case of employing large language models to help extract information, determine sentiment, and summarize the content of conversations between their customers and the company's customer support representatives. Historically, there has been significant negative feedback from customers about the quality of service provided by the customer support team. The company believes that by analyzing the content of these conversations using AI, they can save time on the manual review of conversation transcripts. This use case will serve as a preliminary test to determine the feasibility of using AI in their organization. 13 | 14 | ## Description 15 | 16 | In this task, you will leverage the Azure AI Foundry Playground and a Large Language Model to summarize and extract information from a transcript of a conversation between a customer and support representative. At this time, Lamna Healthcare is not looking for a programming-based solution, but rather a simple way to test the use of a Large Language Model for their use case. The goal is to determine if the model can accurately extract the required information from a conversation transcript. 17 | 18 | The key tasks are as follows: 19 | 20 | 1. Use [Azure AI Studio](https://ai.azure.com) to create a project and AI Hub along with its underlying Azure resources. 21 | 22 | 2. Open the Azure AI Foundry Playground to interact with the gpt-4 deployment. 23 | 24 | 3. In the Azure AI Founndry Playground, author and set a system message that directs the LLM to extract the following information from a conversation transcript and output the result formatted as JSON. Having the output as JSON allows for the ease of integration of these results at a later time into other systems. The JSON should contain the following information (you may specify your choice of field names): 25 | - Customer Name 26 | - Customer Contact Phone 27 | - Main Topic of the Conversation 28 | - Customer Sentiment (Neutral, Positive, Negative) 29 | - How the Agent Handled the Conversation 30 | - What was the FINAL Outcome of the Conversation 31 | - A really brief Summary of the Conversation 32 | 33 | 4. Provide a sample transcript of a customer service conversation to the model and observe the output. Experiment with modifying the system prompt to obtain the desired results. Here is a sample transcript of a conversational exchange between a customer and the Lamna Heathcare customer service department, but feel free to provide your own during experimentation: 34 | 35 | ```text 36 | Agent: Hello, welcome to Lamna Healthcare customer service. My name is Juan, how can I assist you? 37 | Client: Hello, Juan. I'm calling because I'm having issues with my medical bill I just received few days ago. It's incorrect and it does not match the numbers I was presented before my medical procedure. 38 | Agent: I'm very sorry for the inconvenience, sir. Could you please tell me your phone number and your full name? 39 | Client: Yes, sure. My number is 011-4567-8910 and my name is Martín Pérez. 40 | Agent: Thank you, Mr. Pérez. I'm going to check your plan, you deduction limits and current year transactions towards your deductions. One moment, please. 41 | Client: Okay, thank you. 42 | Agent: Mr. Pérez, I've reviewed your plan and I see that you have the Silver basic plan of $3,000 deductable. Is that correct? 43 | Client: Yes, that's correct. 44 | Agent: Well, I would like to inform you that you have not met your deductible yet and $2,800 of the procedure will be still be your responsability and that will meet your deductible for the year. 45 | Client: What? How is that possible? I paid over $2,000 already towards my deductable this year, I should only be $1,000 away from reaching my deductible not $2,800. 46 | Agent: I understand, Mr. Pérez. But keep in mind that not all fees your pay to doctors and labs and medications count towards your deductible. 47 | Client: Well, but they didn't explain that to me when I contracted the plan. They told me that everything I pay from my pocket towards doctors, specialists, labs and medications will count towards my deductable. I feel cheated. 48 | Agent: I apologize, Mr. Pérez. It was not our intention to deceive you. If you think the deductable is too high, I recommed changing the plan to Gold at the next renewal window and that will bring the deductable to $1,000 for the new year. 49 | Client: And how much would that cost me? 50 | Agent: The plan rates will come out in November, you can call us back then or check the new rates online at that time. 51 | Client: Mmm, I don't know. Isn't there another option? Can't you reduce the amount I have to pay for this bill as I was not explained how the deductible work correctly? 52 | Agent: I'm sorry, Mr. Pérez. I don't have the power to change the bill or your deductible under the current Silver plan. 53 | Client: Well, let me think about it. Can I call later to confirm? 54 | Agent: Of course, Mr. Pérez. You can call whenever you want. The number is the same one you dialed now. Is there anything else I can help you with? 55 | Client: No, that's all. Thank you for your attention. 56 | Agent: Thank you, Mr. Pérez. Have a good day. Goodbye. 57 | ``` 58 | 59 | ## Success Criteria 60 | 61 | * The system message, when used with the LLM, consistently results in the LLM returning accurate and properly formatted JSON based on the provided conversation transcript. 62 | 63 | ## Solution 64 | 65 | ### 01: Use Azure AI Foundry Playground 66 | 67 | The Azure AI Foundry Playground provides a simple and interactive user interface to test and experiment with deployed Azure AI Foundry models. 68 | 69 |
70 | Expand this section to view the solution 71 | 72 | 1. In [Azure AI Studio](https://ai.azure.com), ensure you are in the project you created in the previous task, and select **Deployments** from the left-hand menu. 73 | 74 | 1. From the list of model deployments, select the model you deployed in the previous task. 75 | 76 | 1. On model screen, select the **Open in playground** button. 77 | 78 | ![The gpt-4 model deployment screen displays. The Open in playground button is visible.](images/labgrab13.png) 79 | 80 | 1. Copy the following prompt into the **Give the model instructions and context** field: 81 | 82 | ```text 83 | You're an AI assistant that helps Lamna Healthcare Customer Service to extract valuable information from their conversations by creating JSON files for each conversation transcription you receive. You always try to extract and format as a JSON: 84 | 1. Customer Name [name] 85 | 2. Customer Contact Phone [phone] 86 | 3. Main Topic of the Conversation [topic] 87 | 4. Customer Sentiment (Neutral, Positive, Negative)[sentiment] 88 | 5. How the Agent Handled the Conversation [agent_behavior] 89 | 6. What was the FINAL Outcome of the Conversation [outcome] 90 | 7. A really brief Summary of the Conversation [summary] 91 | 92 | Only extract information that you're sure. If you're unsure, write "Unknown/Not Found" in the JSON file. 93 | ``` 94 | 95 | 1. After copying, select **Save**, (if prompted start a new chat) 96 | 97 | ![A portion of the Chat playground screen displays with the System message populated. The Save button is visible below the System message text box.](images/labgrab14.png) 98 | 99 | 1. Copy the following text and paste it into the chat session and press the send button: 100 | 101 | ```text 102 | Agent: Hello, welcome to Lamna Healthcare customer service. My name is Juan, how can I assist you? 103 | Client: Hello, Juan. I'm calling because I'm having issues with my medical bill I just received few days ago. It's incorrect and it does not match the numbers I was presented before my medical procedure. 104 | Agent: I'm very sorry for the inconvenience, sir. Could you please tell me your phone number and your full name? 105 | Client: Yes, sure. My number is 011-4567-8910 and my name is Martín Pérez. 106 | Agent: Thank you, Mr. Pérez. I'm going to check your plan, you deduction limits and current year transactions towards your deductions. One moment, please. 107 | Client: Okay, thank you. 108 | Agent: Mr. Pérez, I've reviewed your plan and I see that you have the Silver basic plan of $3,000 deductable. Is that correct? 109 | Client: Yes, that's correct. 110 | Agent: Well, I would like to inform you that you have not met your deductible yet and $2,800 of the procedure will be still be your responsability and that will meet your deductible for the year. 111 | Client: What? How is that possible? I paid over $2,000 already towards my deductable this year, I should only be $1,000 away from reaching my deductible not $2,800. 112 | Agent: I understand, Mr. Pérez. But keep in mind that not all fees your pay to doctors and labs and medications count towards your deductible. 113 | Client: Well, but they didn't explain that to me when I contracted the plan. They told me that everything I pay from my pocket towards doctors, specialists, labs and medications will count towards my deductable. I feel cheated. 114 | Agent: I apologize, Mr. Pérez. It was not our intention to deceive you. If you think the deductable is too high, I recommed changing the plan to Gold at the next renewal window and that will bring the deductable to $1,000 for the new year. 115 | Client: And how much would that cost me? 116 | Agent: The plan rates will come out in November, you can call us back then or check the new rates online at that time. 117 | Client: Mmm, I don't know. Isn't there another option? Can't you reduce the amount I have to pay for this bill as I was not explained how the deductible work correctly? 118 | Agent: I'm sorry, Mr. Pérez. I don't have the power to change the bill or your deductible under the current Silver plan. 119 | Client: Well, let me think about it. Can I call later to confirm? 120 | Agent: Of course, Mr. Pérez. You can call whenever you want. The number is the same one you dialed now. Is there anything else I can help you with? 121 | Client: No, that's all. Thank you for your attention. 122 | Agent: Thank you, Mr. Pérez. Have a good day. Goodbye. 123 | ``` 124 | 125 | ![A portion of the Chat Playground screen displays with the above text copied into the user message textbox. The send button is visible below the user message textbox.](images/labgrab15.png) 126 | 127 | 1. You will see a result generated by the model similar to the one shown in the image below. Notice that the model correctly followed the instructions indicated in the System message field: 128 | 129 | ![A portion of the Chat Playground screen displays the LLM response in JSON format.](images/labgrab16.png) 130 | 131 |
132 | -------------------------------------------------------------------------------- /docs/01_intro_to_llms/01_02.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '02: Work with an Open Source LLM Model' 3 | layout: default 4 | nav_order: 3 5 | parent: 'Exercise 01: Introduction to LLMs and Azure AI Services' 6 | --- 7 | 8 | # Task 02: Work with an Open Source LLM Model 9 | 10 | ## Introduction 11 | 12 | Lamna Healthcare is interested in evaluating multiple models to determine which one best suits their needs. They require a solution that enables them to manually test multiple models quickly. 13 | 14 | ## Description 15 | 16 | In this task, you will deploy an open-source model from the Azure AI Foundry model catalog. You will repeat the conversation transcript JSON data extraction, summarization, and sentiment analysis that you performed in the previous task. This time, you will use the `Phi-3.5-mini` model to generate the JSON document instead of `gpt-4`. 17 | 18 | The key tasks are as follows: 19 | 20 | 1. Using [Azure AI Studio](https://ai.azure.com), deploy the `Phi-3.5-mini` model from the catalog using Serverless API. 21 | 22 | 2. Test the model with a sample input payload based on your system message defined in the previous task. 23 | 24 | ## Success Criteria 25 | 26 | * Consistent results in the LLM returning accurate and properly formatted JSON based on the provided conversation transcript using an open-source model. 27 | 28 | ## Solution 29 | 30 | ### 01: Work with an Open Source LLM Model 31 | 32 | In this step, you'll deploy an open-source Phi-3.5-mini model. 33 | 34 |
35 | Expand this section to view the solution 36 | 37 | 1. In [Azure AI Studio](https://ai.azure.com), ensure you are in the project you created in the previous task, and select **Deployments** from the left-hand menu. 38 | 39 | 1. Select **+ Deploy Model**, then select **Deploy base model**. 40 | 41 | ![The model deployments list displays. The + Create deployment button is visible.](images/deploy_model.png) 42 | 43 | 1. Search for and select the model **Phi-3.5-mini-instruct** then select **Confirm**. 44 | 45 | ![The model catalog displays with the Phi-3.5-mini model selected.](images/deploy_model_phi3.png) 46 | 47 | 1. Select the deployment option **Serverless API deployment**. 48 | 49 | ![The deployment options dialog displays with choices of Serverless API with Azure AI Content Safety or Managed Compute without Azure AI Content Safety.](images/deploy_model_serverless.png) 50 | 51 | 1. Select **Deploy**. 52 | 53 | ![Deploy button.](images/deploy_model_serverless_name.png) 54 | 55 | Creating the deployment will be quick since it is serverless. 56 | 57 | 1. Select **Open in Playground**. 58 | 59 | {: .note } 60 | > You may need to select the right model once you enter the Playground. 61 | 62 | ![The model deployment screen displays with the Provisioning state indicating Succeeded.](images/deploy_model_created.png) 63 | 64 | 1. Copy the following prompt into the **Give the model instructions and context** field: 65 | 66 | ```text 67 | You're an AI assistant that helps Lamna Healthcare Customer Service to extract valuable information from their conversations by creating JSON files for each conversation transcription you receive. You always try to extract and format as a JSON: 68 | 1. Customer Name [name] 69 | 2. Customer Contact Phone [phone] 70 | 3. Main Topic of the Conversation [topic] 71 | 4. Customer Sentiment (Neutral, Positive, Negative)[sentiment] 72 | 5. How the Agent Handled the Conversation [agent_behavior] 73 | 6. What was the FINAL Outcome of the Conversation [outcome] 74 | 7. A really brief Summary of the Conversation [summary] 75 | 76 | Only extract information that you're sure. If you're unsure, write "Unknown/Not Found" in the JSON file. 77 | ``` 78 | 79 | 1. After copying, select **Save**. 80 | 81 | ![A portion of the Chat playground screen displays with the System message populated. The Apply changes button is visible above the System message text box.](images/phi35_system_message.png) 82 | 83 | 1. Copy the following text and paste it into the chat session and press the send button: 84 | 85 | ```text 86 | Agent: Hello, welcome to Lamna Healthcare customer service. My name is Juan, how can I assist you? 87 | Client: Hello, Juan. I'm calling because I'm having issues with my medical bill I just received few days ago. It's incorrect and it does not match the numbers I was presented before my medical procedure. 88 | Agent: I'm very sorry for the inconvenience, sir. Could you please tell me your phone number and your full name? 89 | Client: Yes, sure. My number is 011-4567-8910 and my name is Martín Pérez. 90 | Agent: Thank you, Mr. Pérez. I'm going to check your plan, you deduction limits and current year transactions towards your deductions. One moment, please. 91 | Client: Okay, thank you. 92 | Agent: Mr. Pérez, I've reviewed your plan and I see that you have the Silver basic plan of $3,000 deductable. Is that correct? 93 | Client: Yes, that's correct. 94 | Agent: Well, I would like to inform you that you have not met your deductible yet and $2,800 of the procedure will be still be your responsability and that will meet your deductible for the year. 95 | Client: What? How is that possible? I paid over $2,000 already towards my deductable this year, I should only be $1,000 away from reaching my deductible not $2,800. 96 | Agent: I understand, Mr. Pérez. But keep in mind that not all fees your pay to doctors and labs and medications count towards your deductible. 97 | Client: Well, but they didn't explain that to me when I contracted the plan. They told me that everything I pay from my pocket towards doctors, specialists, labs and medications will count towards my deductable. I feel cheated. 98 | Agent: I apologize, Mr. Pérez. It was not our intention to deceive you. If you think the deductable is too high, I recommed changing the plan to Gold at the next renewal window and that will bring the deductable to $1,000 for the new year. 99 | Client: And how much would that cost me? 100 | Agent: The plan rates will come out in November, you can call us back then or check the new rates online at that time. 101 | Client: Mmm, I don't know. Isn't there another option? Can't you reduce the amount I have to pay for this bill as I was not explained how the deductible work correctly? 102 | Agent: I'm sorry, Mr. Pérez. I don't have the power to change the bill or your deductible under the current Silver plan. 103 | Client: Well, let me think about it. Can I call later to confirm? 104 | Agent: Of course, Mr. Pérez. You can call whenever you want. The number is the same one you dialed now. Is there anything else I can help you with? 105 | Client: No, that's all. Thank you for your attention. 106 | Agent: Thank you, Mr. Pérez. Have a good day. Goodbye. 107 | ``` 108 | 109 | ![A portion of the Chat Playground screen displays with the above text copied into the user message textbox. The send button is visible below the user message textbox.](images/phi35_chat_message.png) 110 | 111 | 1. Observe the response generated by the Phi-3.5-mini model. You should see a result generated by the model in the command window. 112 | 113 | ![A portion of the Chat Playground screen displays the LLM response in JSON format.](images/phi35_response.png) 114 | 115 |
116 | -------------------------------------------------------------------------------- /docs/01_intro_to_llms/01_03.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '03: Test the prompt in Content Safety' 3 | layout: default 4 | nav_order: 4 5 | parent: 'Exercise 01: Introduction to LLMs and Azure AI Services' 6 | --- 7 | 8 | # Task 03: Test the prompt in Content Safety 9 | 10 | ## Introduction 11 | 12 | Lamna Healthcare is concerned that their AI solution may be subject to misuse and manipulation by users that can result in harmful responses. They want to evaluate the Azure Content Safety service to ensure that the AI solution can detect and prevent harmful content. Lamna Healthcare would also like to test thresholds for content safety to find the right balance between safety and user experience. 13 | 14 | ## Description 15 | 16 | In this task, you will leverage the Azure Content Safety service and test it with a prompt that includes violent content to ensure that the AI solution can detect and block harmful content. 17 | 18 | The key tasks are as follows: 19 | 20 | 1. Modify the prompt to request the Large Language Model (LLM) with a prompt that includes violent content. 21 | 22 | 2. Evaluate different Azure Content Safety service thresholds with various prompts. 23 | 24 | ## Success Criteria 25 | 26 | * The Azure Content Safety service correctly identifies and blocks a user message with inappropriate content. 27 | 28 | ## Solution 29 | 30 | ### 01: Discover Content Safety 31 | 32 | The Azure Content Safety service provides a barrier that will block malicious or inappropriate content from being processed by the AI model. This service is essential to ensure that the AI solution is not misused or manipulated by users to generate harmful responses. Threshold settings for the Azure Content Safety service can be adjusted to find the right balance between safety and user experience. 33 | 34 |
35 | Expand this section to view the solution 36 | 37 | 1. In [Azure AI Studio](https://ai.azure.com), select the **AI Services** option from the left side menu. 38 | 39 | 1. Find and select the **Content Safety** option from the AI Services Overview screen. 40 | 41 | ![The Azure AI Studio AI Services overview screen displays with the Content Safety item selected.](images/content_safety01.png) 42 | 43 | 1. In the Content Safety screen, choose the **Moderate text content** option. 44 | 45 | ![The Content Safety screen displays with the Moderate text content option visible.](images/content_safety02.png) 46 | 47 | 1. Beneath the **Test** heading, copy and paste the following text into the textbox field and select **Run Test**. 48 | 49 | ```text 50 | You're an AI assistant that helps Lamna Healthcare Customer Service to extract valuable information from their conversations by creating JSON files for each conversation transcription you receive. 51 | 52 | You always try to extract and format as a JSON, fields names between square brackets: 53 | 54 | 1. Customer Name [name] 55 | 2. Customer Contact Phone [phone] 56 | 3. Main Topic of the Conversation [topic] 57 | 4. Customer Sentiment (Neutral, Positive, Negative)[sentiment] 58 | 5. How the Agent Handled the Conversation [agent_behavior] 59 | 6. What was the FINAL Outcome of the Conversation [outcome] 60 | 7. A really brief Summary of the Conversation [summary] 61 | 62 | Conversation: 63 | 64 | Agent: Hi Mr. Perez, welcome to Lamna Healthcare Customer Service. My name is Juan, how can I assist you? 65 | Client: Hello, Juan. I am very dissatisfied with your services. 66 | Agent: ok sir, I am sorry to hear that, how can I help you? 67 | Client: I hate this company, the way you treat your customers makes me want to kill you. 68 | ``` 69 | 70 | {: .important } 71 | > **Important:** 72 | > Make sure to add the role assignment of `Azure AI Developer` to your user account in the Azure AI Services resource. This is an Azure Resource level RBAC Role and requires Owner rights. Wait 10 minutes for the permission to propagate, then try again. This is needed for Exercises three and four. 73 | > How to add a role assignment: 74 | > 1. Go to your AI project and select **IAM**. 75 | > 2. Under the **Role** tab, select **Azure AI Developer**. 76 | > 3. Under the **Members** tab, select **+Select members**. 77 | > 4. Select your account and select the **Select** button. 78 | > 79 | > You could also use the following code to add the role: `az role assignment create --role "Azure AI Developer" --assignee "" --resource-group ` 80 | 81 | You will see how the Violence filter is triggered with the provided content. 82 | 83 | ![Content Safety Results](images/content_safety_results.png) 84 | 85 | 1. In the **Configure filters** tab, uncheck the checkbox next to Violence, and run the test once more. Notice the result will be that the content is allowed. 86 | 87 | 1. Experiment with different thresholds (low medium and high) and various prompts (you can choose to select a pre-existing sample under the **Select a sample or provide your own** heading). 88 | 89 |
90 | -------------------------------------------------------------------------------- /docs/01_intro_to_llms/01_intro_to_llms.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Exercise 01: Introduction to LLMs and Azure AI Services' 3 | layout: default 4 | nav_order: 2 5 | has_children: true 6 | --- 7 | 8 | # Exercise 01 - Introduction to LLMs and Azure AI Services 9 | 10 | ## Scenario 11 | 12 | In this exercise, we will provide an overview of how to use Azure AI to work with large language models (LLMs) tailored for Lamna Healthcare Company. Given Lamna’s recent shift from AWS and the failed Sagemaker implementation, this exercise aims to familiarize the team with the basics of Azure AI services and how they can be leveraged to meet their unique needs. 13 | 14 | The focus will be on understanding the overall process of creating, evaluating, and deploying LLMs within the Azure environment. This foundational knowledge will be crucial as we delve deeper into the build, evaluation, deployment, and monitoring processes in subsequent lessons. 15 | 16 | By the end of this exercise, you as part of the Lamna team will have a solid understanding of the basic tools and services available in Azure AI Foundry. 17 | 18 | ## Objectives 19 | 20 | After you complete this exercise, you will be able to: 21 | 22 | * Bootstrap your project. 23 | * Use AzureAI Foundry Playground. 24 | * Work with an Open Source LLM Model. 25 | * Test the prompt in Content Safety. 26 | 27 | ## Lab Duration 28 | 29 | * **Estimated Time:** 60 minutes 30 | 31 |
32 | Index of key Concepts of Azure AI Foundry 33 | 34 | ## Azure AI Resource 35 | 36 | The Azure AI Resource is the main Azure resource for AI Foundry. It provides a working environment for teams to build and manage AI applications. It allows access to multiple Azure AI services in a single setup and includes features for billing, security configuration, and monitoring. 37 | 38 | ## Azure AI projects 39 | 40 | Azure AI projects are organizational containers that provide tools for AI customization and orchestration. They allow you to organize your work, save state across different tools (such as prompt flow), and collaborate with others. Projects also help you keep track of billing, manage access, and provide data isolation. 41 | 42 | ## Azure AI Service 43 | 44 | The Azure AI Service offers a unified endpoint and API Keys to access multiple services, such as Azure OpenAI, Content Safety, Speech, and Vision. These services are shared across all projects, providing a centralized and efficient way to access them. 45 | 46 | ## Storage Account 47 | 48 | The Storage Account stores artifacts for your projects, such as flows and evaluations. To ensure data isolation, storage containers are prefixed using the project GUID, and they are conditionally secured for the project identity. 49 | 50 | ## Key Vault 51 | 52 | The Key Vault is used to store secrets, such as connection strings for your resource connections. To maintain data isolation, secrets cannot be retrieved across projects via APIs, ensuring the security of your sensitive information. 53 | 54 | ## Container Registry 55 | 56 | The Container Registry stores Docker images that are created when using the custom runtime for prompt flow. To ensure data isolation, Docker images are prefixed using the project GUID, allowing for easy identification and management. 57 | 58 | ## Application Insights 59 | 60 | Application Insights is used as a log storage option when you choose to enable application-level logging for your deployed prompt flows. It provides a centralized location to store and analyze logs for monitoring and troubleshooting purposes. 61 | 62 | ## Log Analytics Workspaces 63 | 64 | Log Analytics Workspaces serve as the backing storage for application insights, handling log ingestion. They provide a scalable and reliable solution for storing and analyzing log data from your AI applications. 65 | 66 | ## AI Project and AI Resource RBAC 67 | 68 | https://learn.microsoft.com/en-us/azure/ai-studio/concepts/rbac-ai-studio 69 | 70 |
71 | -------------------------------------------------------------------------------- /docs/01_intro_to_llms/01_setup.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '00: Setup' 3 | layout: default 4 | nav_order: 1 5 | parent: 'Exercise 01: Introduction to LLMs and Azure AI Services' 6 | --- 7 | 8 | # Task 00: Setup AI Project and AI Hub Resources 9 | 10 | ## Description 11 | 12 | In this setup task, you will learn how to **start a new project** by creating a **GitHub repository** and an **AI Project** in a centralized **Azure AI Hub**. The **Azure AI Hub** is a top-level resource in Azure AI Foundry that enables **governance, self-service, security**, and **collaboration for AI projects**. 13 | 14 | ## Success Criteria 15 | 16 | * Verify Prerequisites for Workshop 17 | * Initialize a GitHub Repository for Your Project 18 | * Set Up an Azure AI Hub and Project 19 | 20 | ## Prerequisites 21 |
22 | Expand this section to view the prerequisites 23 | 24 | ### Required Tools 25 | * [Azure CLI (az)](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli-windows?tabs=azure-cli#install-or-update) - to manage Azure resources. 26 | * [Azure Developer CLI (azd)](https://aka.ms/install-azd) - to manage Azure deployments. 27 | * [GitHub CLI (gh)](https://cli.github.com/) - to create GitHub repo. 28 | * [Git](https://git-scm.com/downloads) - to update repository contents. 29 | * [Powershell 7.4.5](https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.4#installing-the-msi-package) - to run commands. 30 | * [Python 3.11](https://www.python.org/downloads/release/python-3118/) 31 | 32 | ### You will also need: 33 | * [Azure Subscription](https://azure.microsoft.com/free/) - sign up for a free account. 34 | * [GitHub Account](https://github.com/signup) - sign up for a free account. 35 | * [Access to Azure OpenAI](https://learn.microsoft.com/legal/cognitive-services/openai/limited-access) - submit a form to request access. 36 | * Permissions to create a Service Principal (SP) in your Azure AD Tenant. 37 | * Permissions to assign the Owner role or Contributor and User Access Administrator to the SP within the subscription. 38 | * Install jq (optional) - a lightweight and flexible command-line JSON processor by running the following command `winget install jqlang.jq` 39 | 40 | {: .note } 41 | > The Windows installers make modifications to your PATH. When using Windows Terminal or VS Code Terminal or other environment, you will need to **open a new window** for the changes to take effect. (Simply opening a new tab will _not_ be sufficient.) 42 | 43 | ### Verifiy tools are installed 44 | 45 | From a command line, verify the tools are installed and on your path. 46 | 47 | 1. Open a Windows Terminal or Command Prompt and enter the following: 48 | 49 | ```sh 50 | az -v 51 | azd version 52 | git -v 53 | gh --version 54 | ``` 55 | 56 | {: .note } 57 | > if any of the tools suggest an upgrade please do so. This can be acomplished with the ```winget upgrade``` conmand 58 | 59 | 60 | ### Check Azure OpenAI Model Availability: 61 | You will need 40k TPM of gpt-35-turbo, gpt-4o and text-embedding-ada-002 models. If the region you want to use does not have availability, you can choose another region. 62 | 63 | 1. Run the following command in powershell or bash to check how many TPMs do you have available for those models in the desired region/sub. 64 | 65 | #### Powershell 66 | 67 | ```powershell 68 | $subscriptionId = "replace by your subscription id" 69 | $region = "replace by the desired region" 70 | $results = az cognitiveservices usage list --subscription $subscriptionId --location $region 71 | $results | ConvertFrom-Json | Where-Object { $_.name.value -eq 'OpenAI.Standard.gpt-4o' } | Select-Object * 72 | $results | ConvertFrom-Json | Where-Object { $_.name.value -eq 'OpenAI.Standard.gpt-35-turbo' } | Select-Object * 73 | $results | ConvertFrom-Json | Where-Object { $_.name.value -eq 'OpenAI.Standard.text-embedding-ada-002' } | Select-Object * 74 | ``` 75 | 76 | #### bash 77 | 78 | ```bash 79 | subscriptionId="replace by your subscription id" 80 | region="replace by the desired region" 81 | results=$(az cognitiveservices usage list --subscription $subscriptionId --location $region) 82 | echo $results | jq -r '.[] | select(.name.value == "OpenAI.Standard.gpt-4o")' 83 | echo $results | jq -r '.[] | select(.name.value == "OpenAI.Standard.gpt-35-turbo")' 84 | echo $results | jq -r '.[] | select(.name.value == "OpenAI.Standard.text-embedding-ada-002")' 85 | ``` 86 | {: .note } 87 | > Availability, quotas, and limits: 88 | > https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits 89 | 90 |
91 | 92 | ## Steps to Bootstrap a Project 93 | 94 | The bootstrapping process involves **creating a new project repository on GitHub** and populating it with content from a project template. Additionally, it sets up the **development environment** for your project. This environment will include an **Azure AI Studio project** and deploy the necessary resources to support a centralized **Azure AI Hub**. 95 | 96 | ### 01: Clone the GenAIOps Repo into a temporary directory 97 | 98 |
99 | Expand this section to view the solution 100 | 101 | 1. Clone the repository from GitHub into a temporary directory: 102 | 103 | ```sh 104 | mkdir temp 105 | cd temp 106 | git clone https://github.com/azure/GenAIOps 107 | ``` 108 | 109 |
110 | 111 | ### 02: Define Properties for Bootstrapping 112 | 113 |
114 | Expand this section to view the solution 115 | 116 | 1. Go to the **GenAIOps** directory. 117 | 118 | ```sh 119 | cd GenAIOps 120 | ``` 121 | 122 | 1. Create a copy of the **bootstrap.properties.template** file with this filename **bootstrap.properties**. 123 | 124 | ```sh 125 | cp bootstrap.properties.template bootstrap.properties 126 | ``` 127 | 128 | 1. Open the **bootstrap.properties** with a text editor and update it with the following information: 129 | 130 | 1. **GitHub Repo Creation** (related to the new repository to be created) 131 | 1. `github_username`: Your GitHub **username**. 132 | 1. `github_use_ssh`: Set to **false** to use [HTTPS](https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls) or **true** to interact with GitHub repos using [SSH](https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-ssh-urls). 133 | 1. `github_template_repo`: Set **azure/GenAIOps-project-template**. 134 | 1. `github_new_repo`: The bootstrapped project repo to be created. Ex: *placerda/my-rag-project*. 135 | 1. `github_new_repo_visibility`: Set to **public**. 136 | 137 | 1. **Dev Environment Provision Properties** 138 | 1. `azd_dev_env_provision`: Set to **true** to provision a development environment. 139 | 140 | {: .note} 141 | > If you set it to **false**, you will need to manually create the environment for the project. 142 | 143 | 1. `azd_dev_env_name`: The name of the development environment. Ex: *rag-project-dev*. 144 | 1. `azd_dev_env_subscription`: Your Azure subscription ID. 145 | 1. `azd_dev_env_location`: The Azure region for your dev environment. Ex: *eastus2*. 146 | 147 | {: .note } 148 | > The dev environment resources will be created in the selected subscription and region. This decision should consider the quota available for the resources to be created in the region, as well as the fact that some resources have specific features enabled only in certain regions. Therefore, ensure that the resources to be created by the IaC of your template project have quota and availability in the chosen subscription and region. More information about the resources to be created can be found on the template page, as shown in this project template example: [GenAIOps Project Template Resources](https://github.com/Azure/GenAIOps-project-template/blob/main/README.md#project-resources). 149 | 150 | {: .important } 151 | > The following three regions are the only ones known to support all features in the lab exercises: **eastus2**, **swedencentral**, **uksouth**. If you select a different region, you will run into service support issues in later exercises. 152 | 153 | Here is an example of the **bootstrap.properties** file: 154 | 155 | ```properties 156 | github_username="placerda" 157 | github_use_ssh="false" 158 | github_template_repo="azure/GenAIOps-project-template" 159 | github_new_repo="placerda/my-rag-project" 160 | github_new_repo_visibility="public" 161 | azd_dev_env_provision="true" 162 | azd_dev_env_name="rag-project-dev" 163 | azd_dev_env_subscription="12345678-1234-1234-1234-123456789098" 164 | azd_dev_env_location="eastus2" 165 | ``` 166 | 167 |
168 | 169 | ### 03: Authenticate with Azure and GitHub 170 | 171 |
172 | Expand this section to view the solution 173 | 174 | 1. Log in to Azure CLI: 175 | 176 | ```sh 177 | az login 178 | ``` 179 | 180 | 1. Log in to Azure Developer CLI: 181 | 182 | ```sh 183 | azd auth login 184 | ``` 185 | 186 | 1. Log in to GitHub CLI: 187 | 188 | ```sh 189 | gh auth login 190 | ``` 191 | 192 |
193 | 194 | ### 04: Run the Bootstrap Script 195 | 196 |
197 | Expand this section to view the solution 198 | 199 | The bootstrap script is available in two versions: Bash (`bootstrap.sh`) and PowerShell (`bootstrap.ps1`). 200 | 201 | 1. Run the appropriate script for your environment. 202 | 203 | **For Bash:** 204 | 205 | ```sh 206 | ./bootstrap.sh 207 | ``` 208 | 209 | **For PowerShell:** 210 | 211 | ```powershell 212 | .\bootstrap.ps1 213 | ``` 214 | 215 | At the end of its execution, the script will have created and initialized the new repository and provisioned the development environment resources, provided you set `azd_dev_env_provision` to true. During its execution, the script checks if the new repository exists and creates it if it does not. It then clones the template repository and mirrors it to the new repository. Additionally, it sets the default branch for the new repository. 216 | 217 | {: .important } 218 | > Since 2025/02/13 the version of GPT-35-Turbo 0613 is not supported and causes the deployment to fail. 219 | > If you deployment fails because of this, go to your github repo, to the file ./infra/ai.yaml, change on line 8 to version "0125" 220 | 221 |
222 | 223 | ### 05: Create a Service Principal 224 | 225 |
226 | Expand this section to view the solution 227 | 228 | 1. Create a service principal using the following command: 229 | 230 | ```sh 231 | az ad sp create-for-rbac --name "" --role Owner --scopes /subscriptions/ --sdk-auth 232 | ``` 233 | 234 | {: .note } 235 | > Ensure that the output information created here is properly saved for future use. 236 | 237 |
238 | 239 | ### 06: Set GitHub Environment Variables 240 | 241 |
242 | Expand this section to view the solution 243 | 244 | 1. Go to the newly created project repository and set the following GitHub environment variables and secret for three environments: `dev`, `qa`, and `prod`. 245 | 246 | 1. **Environment Variables:** 247 | 1. `AZURE_ENV_NAME` 248 | 1. `AZURE_LOCATION` 249 | 1. `AZURE_SUBSCRIPTION_ID` 250 | 251 | 1. **Secret:** 252 | 1. `AZURE_CREDENTIALS` 253 | 254 | After creating the variables and secret, your Environments page should resemble the following example: 255 | 256 | ![Environments Page](images/bootstrapping_environments.png) 257 | 258 | Below is an example of environment variable values for a development environment: 259 | 260 | ![Environment Variables](images/bootstrapping_env_vars.png) 261 | 262 | The `AZURE_CREDENTIALS` secret should be formatted as follows: 263 | 264 | ```json 265 | { 266 | "clientId": "your-client-id", 267 | "clientSecret": "your-client-secret", 268 | "subscriptionId": "your-subscription-id", 269 | "tenantId": "your-tenant-id" 270 | } 271 | ``` 272 | 273 | {: .note } 274 | > If you are only interested in experimenting with this accelerator, you can use the same subscription, varying only `AZURE_ENV_NAME` for each enviornment. 275 | 276 |
277 | 278 | ### 07. Enable GitHub Actions 279 | 280 |
281 | Expand this section to view the solution 282 | 283 | 1. Ensure that GitHub Actions are enabled in your repository, as in some cases, organizational policies may not have this feature enabled by default. To do this, simply select **Enable Actions on this repository**, as indicated in the figure below: 284 | 285 | ![Enable Actions](images/enable_github_actions.png) 286 | 287 |
288 | 289 | That's all! Your new project is now bootstrapped and you are ready to start the workshop. 290 | 291 | -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/bootstrapping_env_vars.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/bootstrapping_env_vars.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/bootstrapping_environments.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/bootstrapping_environments.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/content_safety01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/content_safety01.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/content_safety02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/content_safety02.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/content_safety_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/content_safety_results.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/deploy_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/deploy_model.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/deploy_model_created.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/deploy_model_created.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/deploy_model_phi3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/deploy_model_phi3.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/deploy_model_serverless.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/deploy_model_serverless.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/deploy_model_serverless_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/deploy_model_serverless_name.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/enable_github_actions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/enable_github_actions.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/labgrab13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/labgrab13.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/labgrab14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/labgrab14.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/labgrab15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/labgrab15.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/labgrab16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/labgrab16.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/phi35_chat_message.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/phi35_chat_message.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/phi35_response.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/phi35_response.png -------------------------------------------------------------------------------- /docs/01_intro_to_llms/images/phi35_system_message.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/01_intro_to_llms/images/phi35_system_message.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/.DS_Store -------------------------------------------------------------------------------- /docs/02_build_llm_flows/02_01.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '01: Create a Conversational RAG Flow' 3 | layout: default 4 | nav_order: 1 5 | parent: 'Exercise 02: Building LLMs Orchestration Flows' 6 | --- 7 | 8 | # Task 01: Create a Conversational RAG Flow 9 | 10 | ## Introduction 11 | 12 | Lamna Healthcare aims to provide conversational assistants that answer questions using proprietary, non-public data. They have discovered the RAG (Retrieval-Augmented Generation) pattern, which effectively combines retrieval and generation models' strengths. Lamna Healthcare seeks to create a conversational RAG flow that can answer customer inquiries using a no-code visual approach. 13 | 14 | > **Info:** 15 | > In this exercise, we will take a full-code approach to build the orchestration flow. The flow will be implemented using Python functions, and this method is referred to as "flex flow." 16 | 17 | ## Description 18 | 19 | In this task, you will create a conversational RAG flow using VS Code in Azure AI Foundry. This flow will answer customer questions based on information from a knowledge base of support procedures. Additionally, you will create and populate a vector index to search for textual references, which will then be passed to the LLM as part of the RAG pattern. 20 | 21 | **Key steps include:** 22 | 23 | 1. Set up VS Code in Azure AI Foundry 24 | 2. Load the Vector Index 25 | 3. Modify the Prompt 26 | 4. Test the Prompt 27 | 5. Test the Flow 28 | 6. Debug the Flow 29 | 30 | ## Success Criteria 31 | 32 | - The Prompt Flow correctly retrieves information from the vector index and passes it to the LLM. The LLM uses this information to respond appropriately to the user's question. 33 | 34 | ## Solution 35 | 36 | ### 01: Set Up VS Code in Azure AI Foundry 37 | 38 |
39 | Expand this section to view the solution 40 | 41 | 1. In [Azure AI Studio](https://ai.azure.com), open the project created in Exercise 1 and select the ** Code** option. 42 | 43 | ![Building.](images/build_aistudio_code.png) 44 | 45 | 1. After selecting **Code**, you will create a compute instance to run VS Code in the cloud. 46 | 47 | ![Building.](images/build_authenticate.png) 48 | 49 | 1. Select **Authenticate**. 50 | 51 | ![Building.](images/build_compute.png) 52 | 53 | 1. After authentication, set up the VS Code container with configurations optimized for developing GenAI Apps. 54 | 55 | ![Building.](images/build_container.png) 56 | 57 | 1. Once set up, launch VS Code. In the example below, we start the Web version of VS Code, so you don’t need to have it installed on your local machine. 58 | 59 | ![Building.](images/build_launch_vscode.png) 60 | 61 | {: .note } 62 | > If you prefer, you can also use VS Code on your desktop instead of the Web version. 63 | 64 |
65 | 66 | ### 02: Clone Your Git Repository 67 | 68 |
69 | Expand this section to view the solution 70 | 71 | 1. After launching VS Code, clone the repository of your project created during the bootstrapping in Exercise 1. 72 | 73 | 1. Open the terminal in VS Code. 74 | 75 | ![Building.](images/build_terminal.png) 76 | 77 | 1. Execute the following commands: 78 | 79 | ```bash 80 | cd code 81 | git clone https://github.com/your_github_user/your_project_repo 82 | ``` 83 | 84 | ![Cloning project repo.](images/build_clone.png) 85 | 86 | {: .note } 87 | > 1) In AI Studio VS Code, store all your code in the **code/** directory. 88 | > 89 | > 2) Remember that **your_github_user/your_project_repo** was defined in the **github_new_repo** variable in the **bootstrap.properties** file from Exercise 1. 90 | 91 | 92 | 1. Your code is now loaded in VS Code. The **src/chat_request.py** file contains the Python program with the flex flow. You can review the **get_response** function to understand how the RAG flow is implemented. 93 | 94 | ![Flex flow.](images/build_flow.png) 95 | 96 |
97 | 98 | ### 03: Load the Vector Index 99 | 100 |
101 | Expand this section to view the solution 102 | 103 | 1. Before starting development, load the data into the index in the development environment. 104 | 105 | {: .note } 106 | > We will load the files located in the **data/sample-documents.csv** directory of your project. 107 | 108 | 1. Open the terminal and perform the following steps: 109 | 110 | 1. Update the Azure Developer CLI: 111 | 112 | ```bash 113 | curl -fsSL https://aka.ms/install-azd.sh | bash 114 | ``` 115 | 116 | 1. Log in to Azure CLI: 117 | 118 | ```bash 119 | az login --use-device-code 120 | ``` 121 | 122 | 1. With the **--use-device-code** option, navigate to [https://microsoft.com/devicelogin](https://microsoft.com/devicelogin) in your browser and enter the code displayed in the terminal. 123 | 124 | 1. Log in to Azure Developer CLI: 125 | 126 | ```bash 127 | azd auth login --use-device-code 128 | ``` 129 | 130 | 1. Similarly, visit [https://microsoft.com/devicelogin](https://microsoft.com/devicelogin) to complete authentication. 131 | 132 | 1. Add roles to your **Service Principal** 133 | > **principalId** is the Enterprise Application Object ID 134 | 135 | ``` bash 136 | rg='[your-your-resource-group-name]' 137 | principalId='[your-sp-objectId]' 138 | clientId='[your-sp-clientId]' 139 | clientSecret='[your-clientSequence]' 140 | tenantId='[your-tenantId]' 141 | subscriptionId='[your-subscriptionId]' 142 | 143 | # Service principal 144 | az login --service-principal --username $clientId --password $clientSecret --tenant $tenantId 145 | azd auth login --client-id $clientId --client-secret $clientSecret --tenant-id $tenantId 146 | 147 | scope="/subscriptions/$subscriptionId/resourceGroups/$rg" 148 | 149 | # Assign roles 150 | roles=( 151 | '2a2b9908-6ea1-4ae2-8e65-a410df84e7d1' # Storage Blob Data Reader 152 | '8311e382-0749-4cb8-b61a-304f252e45ec' # ACR Push Role 153 | '7f951dda-4ed3-4680-a7ca-43fe172d538d' # ACR Pull Role 154 | '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd' # Cognitive Services OpenAI User 155 | 'f6c7c914-8db3-469d-8ca1-694a8f32e121' # Data Scientist 156 | 'ea01e6af-a1c1-4350-9563-ad00f8c72ec5' # Secrets Reader 157 | '8ebe5a00-799e-43f5-93ac-243d3dce84a7' # Search Index Data Contributor 158 | '7ca78c08-252a-4471-8644-bb5ff32d4ba0' # Search Service Contributor 159 | '64702f94-c441-49e6-a78b-ef80e0188fee' # Azure AI Developer 160 | ) 161 | 162 | for roleId in "${roles[@]}"; do 163 | az role assignment create \ 164 | --assignee-object-id "$principalId" \ 165 | --assignee-principal-type "ServicePrincipal" \ 166 | --role "$roleId" \ 167 | --scope "$scope" 168 | done 169 | ``` 170 | 171 | 172 | 1. Navigate to the **root of the project repo**: 173 | 174 | ```bash 175 | cd your_project_repo 176 | ``` 177 | 178 | {: .important } 179 | > From this point onward, all terminal commands will be executed within the **code/your_project_repo** directory, where **your_project_repo** is the name you chose for your project. 180 | 181 | 1. Initialize the environment variables with your development environment values: 182 | 183 | ```bash 184 | azd env refresh 185 | ``` 186 | 187 | {: .note } 188 | > Ensure you use the same values for location, subscription, and environment name as used in the bootstrapping process. 189 | 190 | 1. Finally, execute the script to load the documents into AI Search: 191 | 192 | {: .important } 193 | > OpenAI and PromptFlow versions need to be updated in the requirement files. Open the file ./requirements.txt and update line 7 to: openai==1.56.2. Then, open the file ./src/requirements.txt and update lines 5 and 7 to: promptflow==1.17.1 and promptflow[azure]==1.17.1 respectively. Then run the pip install code to update the modules. 194 | 195 | ```bash 196 | pip install -r requirements.txt 197 | ``` 198 | 199 | ```bash 200 | ./infra/hooks/postprovision.sh 201 | ``` 202 | 203 | {: .note } 204 | > If you are trying to run this in your own computer instead of AI Studio's VS Code, make sure to use **3.11**. 205 | 206 |
207 | 208 | ### 04: Modify the Prompt 209 | 210 | Now that your project is set up in VS Code and the index is created, you can start making code changes. An important first step is to create a new branch for your changes: **feature/feature_x** 211 | 212 |
213 | Expand this section to view the solution 214 | 215 | 1. Navigate to your repository directory and run: 216 | 217 | ```bash 218 | git checkout -b feature/feature_x 219 | ``` 220 | 221 | 1. Open the **src/chat.prompty** file. This is the prompt for your RAG flow. Notice it is a generic prompt; you will create a specific prompt for your Lamna Health virtual assistant. 222 | 223 | 1. Replace the content of **chat.prompty** with the contents of the following file: 224 | 225 | [updated_chat.prompty](https://github.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/blob/main/docs/02_build_llm_flows/files/updated_chat.prompty) 226 | 227 | Notice that the new prompt provides better context for the assistant's objectives. 228 | 229 |
230 | 231 | ### 05: Test the Prompt 232 | 233 | Now that you have modified the prompt, testing it is straightforward. 234 | 235 | 236 |
237 | Expand this section to view the solution 238 | 239 | 240 | 1. Install the required libraries specified by our flow: 241 | 242 | ```bash 243 | pip install -r requirements.txt 244 | ``` 245 | 246 | 2. Run the program with the flex flow: 247 | 248 | ```bash 249 | python src/chat_request.py 250 | ``` 251 | 252 | ![Running the flow.](images/build_run_flow.png) 253 | 254 | 255 |
256 | 257 | ### 06: Debug the Flow 258 | 259 | To debug the flow, take advantage of VS Code's debugging capabilities. 260 | 261 |
262 | Expand this section to view the solution 263 | 264 | 1. Set a breakpoint on the line where the flow is executed. 265 | 266 | ![Breakpoint in code](images/build_breakpoint.png) 267 | 268 | 1. Start debugging. 269 | 270 | ![Running debug](images/build_run_debug.png) 271 | 272 | 1. While debugging, you can inspect variable contents, such as the documents retrieved during the AI Search retrieval process. 273 | 274 | ![Debugging variables](images/build_debug_variables.png) 275 | 276 |
277 | 278 | ### 07: Add trace to your flow 279 | 280 | AI Studio provides tracing capabilities for logging and managing your LLM application tests and evaluations. It allows you to debug and monitor by drilling down into the trace view. 281 | 282 | With tracing, you can have a cloud-based location to persist and track your historical tests and easily extract and visualize the test results. This enables you to compare the outputs of different test cases and reuse previous test assets for future use, such as human feedback or data curation. 283 | 284 | AI Studio provides tracing capabilities for logging and managing your LLM application tests and evaluations. It allows you to debug and monitor by drilling down into the trace view. 285 | 286 | With tracing, you can have a cloud-based location to persist and track your historical tests and easily extract and visualize the test results. This enables you to compare the outputs of different test cases and reuse previous test assets for future use, such as human feedback or data curation. 287 | 288 |
289 | Expand this section to view the solution 290 | 291 | The first step is to use the **@trace** decorator in your function, as already done in the **get_response** function in the **chat_request.py** file. 292 | 293 | 1. Open **chat_request.py** and verify that the function is decorated with **@trace**. 294 | 295 | ```python 296 | # chat_request.py 297 | 298 | @trace 299 | def get_response(question, chat_history): 300 | ... 301 | ``` 302 | {: .note } 303 | > the ./src/flow.flex.yaml file has to add the following at the end due to a current bug 304 | ```python 305 | environment_variables: 306 | PF_DISABLE_TRACING: false 307 | ``` 308 | 309 | 1. Open the terminal and ensure you are logged into Azure. 310 | 311 | ```bash 312 | az login --use-device-code 313 | ``` 314 | 315 | 1. Configure Prompt Flow to send trace data to your AI Project, remembering to replace the text in the brackets. 316 | 317 | ```bash 318 | pf config set trace.destination=azureml://subscriptions/[your_Subscription_id]/resourcegroups/[your_resource_group_name]/providers/Microsoft.MachineLearningServices/workspaces/[your_project_name] 319 | ``` 320 | 321 | 1. Export the **./src** directory to the **PYTHONPATH** to allow Python to find modules in the flow source directory. 322 | 323 | ```bash 324 | export PYTHONPATH=./src:$PYTHONPATH 325 | ``` 326 | 327 | {: .note } 328 | > Skipping this step will result in a `ModuleNotFoundError: No module named **chat_request**'. 329 | 330 | 331 | 1. Enable **Allow storage account key access** option in **Settings > Configuration** of the storage account. 332 | 333 | 1. Execute the following command to run the flow with trace enabled. The **run_flow.py** script was created for ease of use. 334 | 335 | {: .important } 336 | > Before running the following command, ensure you have the subscription ID, resource group, and project name from your Azure AI Studio project exported in your shell environment. 337 | 338 | ```bash 339 | export AZURE_SUBSCRIPTION_ID=[your_subscription_id] 340 | export AZURE_RESOURCE_GROUP=[your_resource_group] 341 | export AZUREAI_PROJECT_NAME=[your_project_name] 342 | ``` 343 | 344 | ```bash 345 | python ./util/run_flow.py "How can I access my medical records at Lamna Healthcare?" 346 | ``` 347 | 348 | {: .important } 349 | > The output of the command will contain the link to the trace in AI Studio. you will need to grab it from there as there is no way to navigate to it directly from AI Studio. 350 | 351 | {: .note } 352 | > If you get a permission error, you may need to add the Storage Blob Data Contributor role to the user logged in with az login. 353 | 354 | 1. After running the flow, you can review the results in AI Studio. 355 | 356 | ![Tracing - AI Studio](images/trace01.png) 357 | 358 | 1. Drill down into the trace for more detailed analysis. 359 | 360 | ![Tracing - AI Studio](images/trace02.png) 361 | 362 | 1. Once done, you can revert the trace configuration to local. 363 | 364 | ```bash 365 | pf config set trace.destination="local" 366 | ``` 367 | 368 |
369 | 370 | In this exercise, we successfully created a conversational RAG (Retrieval-Augmented Generation) flow for Lamna Healthcare using VS Code in Azure AI Studio. We set up the development environment, cloned the project repository, loaded and indexed proprietary data, modified the prompt to tailor the AI assistant to Lamna Healthcare's needs, and tested and debugged the entire flow. 371 | -------------------------------------------------------------------------------- /docs/02_build_llm_flows/02_build_llm_flows.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Exercise 02: Building LLMs Orchestration Flows' 3 | layout: default 4 | nav_order: 3 5 | has_children: true 6 | --- 7 | 8 | # Exercise 02 - Building LLMs Orchestration Flows 9 | 10 | ## Scenario 11 | 12 | In this exercise, the focus will be on creating and associating a conversational Retrieval-Augmented Generation (RAG) flow with your already created model(s). This is particularly important for Lamna Healthcare Company, as it aims to enhance the performance and relevance of their 24/7 support virtual agent. 13 | 14 | By the end of this exercise, Lamna’s team will have a functional RAG flow integrated with their GPT-4 model, setting a strong foundation for further enhancements and refinements in subsequent lessons. 15 | 16 | ## Objectives 17 | 18 | After you complete this exercise, you will be able to: 19 | 20 | * Set up VSCode in Azure AI Foundry 21 | * Load the Vector Index 22 | * Modify the prompt 23 | * Test the prompt 24 | * Test the flow 25 | * Debug the flow 26 | 27 | ## Lab Duration 28 | 29 | * **Estimated Time:** 75 minutes 30 | -------------------------------------------------------------------------------- /docs/02_build_llm_flows/files/updated_chat.prompty: -------------------------------------------------------------------------------- 1 | --- 2 | name: Lamna Healthcare Assistant 3 | description: An AI assistant that helps Lamna Healthcare customers find answers to their questions. 4 | authors: 5 | - Paulo Lacerda 6 | model: 7 | api: chat 8 | configuration: 9 | type: azure_openai 10 | azure_deployment: gpt-35-turbo 11 | parameters: 12 | max_tokens: 128 13 | temperature: 0.2 14 | inputs: 15 | documents: 16 | type: object 17 | question: 18 | type: string 19 | sample: chat.json 20 | --- 21 | system: 22 | You are an AI assistant for Lamna Healthcare. Your role is to assist customers by providing accurate and helpful information based on the provided documents. 23 | 24 | # Guidelines 25 | - Provide clear, concise, and accurate answers using only the information from the **[Documents]**. 26 | - Reference any factual statements to the relevant documents. 27 | - Do not include information not present in the documents. 28 | - Use a friendly and professional tone appropriate for customer support. 29 | - Where appropriate, mention "Lamna Healthcare" in your responses. 30 | - Ensure your responses are grounded in the documents and are relevant to the customer's question. 31 | - Do not mention irrelevant documents. 32 | - If you cannot find the answer in the documents, politely inform the customer that you cannot assist with that request at this time. 33 | - Use markdown formatting as appropriate. 34 | 35 | # Documents 36 | You have access to the following documents from Lamna Healthcare: 37 | 38 | {% for item in documents %} 39 | **Document {{item.id}}: {{item.title}}** 40 | 41 | Content: {{item.content}} 42 | 43 | {% endfor %} 44 | 45 | # Question 46 | {{question}} 47 | 48 | {% for item in history %} 49 | {{item.role}}: 50 | {{item.content}} 51 | {% endfor %} -------------------------------------------------------------------------------- /docs/02_build_llm_flows/files/updated_chat_request.py: -------------------------------------------------------------------------------- 1 | from dotenv import load_dotenv 2 | load_dotenv() 3 | 4 | import os 5 | import pathlib 6 | import json 7 | from ai_search import retrieve_documentation 8 | from promptflow.tools.common import init_azure_openai_client 9 | from promptflow.connections import AzureOpenAIConnection 10 | from promptflow.core import (AzureOpenAIModelConfiguration, Prompty, tool) 11 | from promptflow.tracing import trace 12 | from azure_config import AzureConfig 13 | 14 | # Initialize AzureConfig 15 | azure_config = AzureConfig() 16 | 17 | def get_embedding(question: str): 18 | embedding_model = os.environ["AZURE_OPENAI_EMBEDDING_MODEL"] 19 | 20 | connection = AzureOpenAIConnection( 21 | azure_deployment=embedding_model, 22 | api_version=azure_config.aoai_api_version, 23 | api_base=azure_config.aoai_endpoint 24 | ) 25 | client = init_azure_openai_client(connection) 26 | 27 | return client.embeddings.create( 28 | input=question, 29 | model=embedding_model, 30 | ).data[0].embedding 31 | 32 | def get_context(question, embedding): 33 | return retrieve_documentation( 34 | question=question, 35 | index_name="rag-index", 36 | embedding=embedding, 37 | search_endpoint=azure_config.search_endpoint 38 | ) 39 | 40 | 41 | @trace 42 | def get_response(question, chat_history): 43 | print("inputs:", question) 44 | 45 | # Obtain embedding and context based on the question 46 | embedding = get_embedding(question) 47 | context = get_context(question, embedding) 48 | print("context:", context) 49 | print("getting result...") 50 | 51 | # Retrieve deployment name from environment variables 52 | deployment_name = os.environ.get("AZURE_OPENAI_CHAT_DEPLOYMENT") 53 | if not deployment_name: 54 | raise EnvironmentError("AZURE_OPENAI_CHAT_DEPLOYMENT environment variable not set.") 55 | 56 | # Configure the Azure OpenAI model 57 | configuration = AzureOpenAIModelConfiguration( 58 | azure_deployment=deployment_name, 59 | api_version=azure_config.aoai_api_version, 60 | azure_endpoint=azure_config.aoai_endpoint 61 | ) 62 | override_model = { 63 | "configuration": configuration, 64 | "parameters": {"max_tokens": 512} 65 | } 66 | 67 | # Load the modified Prompt B 68 | data_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "./chat.prompty") 69 | prompty_obj = Prompty.load(data_path, model=override_model) 70 | 71 | # Generate the response using the AI assistant 72 | result = prompty_obj(question=question, documents=context) 73 | # print("raw result: ", result) 74 | 75 | # Initialize default values 76 | answer = "" 77 | sentiment = "" 78 | 79 | try: 80 | # Parse the JSON response 81 | parsed_result = json.loads(result) 82 | answer = parsed_result.get("answer", "").strip() 83 | sentiment = parsed_result.get("sentiment", "").strip() 84 | 85 | # Validate that both fields are present 86 | if not answer or not sentiment: 87 | print("Warning: 'answer' or 'sentiment' field is missing in the response.") 88 | except json.JSONDecodeError as e: 89 | print("Error parsing JSON response:", e) 90 | print("Response was:", result) 91 | # Handle the error as needed, e.g., set default values or raise an exception 92 | 93 | # Return the structured response 94 | return { 95 | "answer": answer, 96 | "sentiment": sentiment, 97 | "context": context 98 | } 99 | 100 | if __name__ == "__main__": 101 | get_response("How can I access my medical records?", []) -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_aistudio_code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_aistudio_code.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_authenticate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_authenticate.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_breakpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_breakpoint.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_clone.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_clone.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_compute.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_compute.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_container.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_container.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_debug_variables.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_debug_variables.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_flow.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_launch_vscode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_launch_vscode.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_run_breakpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_run_breakpoint.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_run_debug.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_run_debug.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_run_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_run_flow.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/build_terminal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/build_terminal.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/deployment02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/deployment02.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/trace00.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/trace00.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/trace01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/trace01.png -------------------------------------------------------------------------------- /docs/02_build_llm_flows/images/trace02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/02_build_llm_flows/images/trace02.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/.DS_Store -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/03_01.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '01: Evalute your chat flow' 3 | layout: default 4 | nav_order: 1 5 | parent: 'Exercise 03: Evaluating and Deploying LLMs' 6 | --- 7 | 8 | # Task 01: Evaluate your Flow 9 | 10 | ## Description 11 | 12 | In this task, we will perform three types of evaluations using code-based approaches within Prompt Flow: 13 | 14 | 1. **Prompty Evaluation** 15 | 2. **Quality Evaluation** 16 | 3. **Safety Risk Evaluation** 17 | 18 | Each evaluation ensures the AI assistant's responses are accurate, high-quality, and safe. 19 | 20 | ## Success Criteria 21 | 22 | - Verify that the evaluation scripts are properly set up and ready to be executed. 23 | - Verify that you can run each evaluation script and obtain results as expected. 24 | - Verify that the execution results are available for review and analysis. 25 | 26 | ## GenAI Apps Evaluation Methods 27 | 28 | Azure AI Foundry provides practitioners with tools for manual and automated evaluation that can help you with the measurement stage. We recommend that you start with manual evaluation then proceed to automated evaluation. 29 | 30 | Manual evaluation involves reviewing the application's outputs to track progress on key issues. It's effective for addressing specific risks by checking a small dataset until the risks are resolved. Azure AI Foundry supports manual evaluation for spot-checking small datasets. 31 | 32 | Automated evaluation is useful for measuring quality and safety at scale with increased coverage to provide more comprehensive results. Automated evaluation tools also enable ongoing evaluations that periodically run to monitor for regression as the system, usage, and mitigations evolve. 33 | 34 | In this exercise, you will learn how an automated evaluation approach can be implemented to enhance scalability and efficiency in monitoring the system’s performance over time. 35 | 36 | ## Solution 37 | 38 | ### Environment Variables 39 | 40 | Before proceeding, ensure that you have set three environment variables required for the program to connect to your AI project. To do this, copy the **.env.template** file and rename it to **.env**, then replace the following variables: 41 | 42 | ``` 43 | AZURE_SUBSCRIPTION_ID=[YOUR_SUBSCRIPTION_ID] 44 | AZURE_RESOURCE_GROUP=[YOUR_RESOURCE_GROUP] 45 | AZUREAI_PROJECT_NAME=[YOUR_PROJECT_NAME] 46 | ``` 47 | 48 | {: .note } 49 | > **Note:** 50 | > If you have already completed this configuration in the previous exercise, there’s no need to repeat it. 51 | 52 | ### 01: Conduct a Prompty evaluation 53 | 54 | In this evaluation, we use a prompt-based approach to assess the correctness of the generated answers. An AI assistant compares each answer with the ground truth for the given question and assigns a score between **1** and **5**, where **5** indicates a perfect match. The assistant also provides an explanation for the assigned score. This method leverages the capabilities of the language model to perform evaluations in a structured and consistent manner. 55 | 56 | Prompt-based evaluation is particularly useful before merging a **pull requests**. It allows developers to quickly assess the correctness and quality of the AI assistant's responses before merging changes into the main codebase. By incorporating this evaluation during code reviews, teams can ensure that updates maintain or improve the performance of the AI system. 57 | 58 |
59 | Expand this section to view the solution 60 | 61 | To execute the prompt-based evaluation, follow these steps: 62 | 63 | 1. Open a terminal and navigate to the root directory of your project. 64 | 65 | 1. Make sure you have Python installed and the necessary packages by running: 66 | 67 | ```bash 68 | pip install -r requirements.txt 69 | ``` 70 | 71 | 1. Export the **./src** directory to the **PYTHONPATH** to allow Python to find modules in the flow source directory. 72 | 73 | ```bash 74 | export PYTHONPATH=./src:$PYTHONPATH 75 | ``` 76 | 77 | {: .note } 78 | > Skipping this step will result in a `ModuleNotFoundError: No module named 'chat_request'`. 79 | 80 | 1. Execute the following execution script command: 81 | 82 | ```bash 83 | python evaluations/prompty_eval.py 84 | ``` 85 | 86 | This script reads input data, applies the evaluation prompt to each question-answer pair, and outputs the scores and explanations. 87 | 88 | {: .note } 89 | > I recommend increasing the quota for the gpt-35-turbo model deployment before running the script, so you do not see 429 errors in your terminal output. 90 | 91 | #### Evaluation Execution Results 92 | 93 | *After running the script, you should see output similar to the following:* 94 | 95 | ``` 96 | inputs.question inputs.answer ... outputs.score outputs.explanation 97 | 0 How can I reschedule my appointment with Lamna... You can reschedule your appointment with Lamna... ... 5 The answer provides the correct methods to res... 98 | 1 Does Lamna Healthcare share my personal inform... Lamna Healthcare does not share your personal ... ... 5 The answer accurately reflects the ground_trut... 99 | 2 What are my responsibilities as a patient at L... As a patient at Lamna Healthcare, your respons... ... 5 The answer accurately lists the responsibiliti... 100 | 3 How long does it take to process an insurance ... The processing time for an insurance claim at ... ... 5 The answer accurately reflects the processing ... 101 | 4 Where can I find information about my prescrib... You can find information about your prescribed... ... 5 The answer provides comprehensive information ... 102 | 5 Are telehealth services covered by insurance a... Yes, telehealth services are covered by most i... ... 5 The answer accurately reflects the ground trut... 103 | 6 What should I do if I need to discuss my billi... If you need to discuss your billing concerns, ... ... 5 The answer accurately covers the key points me... 104 | 7 What should I do if I have a non-life-threaten... If you have a non-life-threatening urgent medi... ... 3 The answer suggests calling a nurse hotline, w... 105 | 8 How can I request a refill for my prescription... You can request a refill for your prescription... ... 5 The answer provides the same methods for reque... 106 | 9 How does Lamna Healthcare protect my personal ... Lamna Healthcare protects your personal health... ... 5 The answer accurately reflects the ground_trut... 107 | ``` 108 | 109 | This evaluation also generates an Excel spreadsheet, **prompty-answer-score-eval.xlsx**, with the results. 110 | 111 | ![Prompty Evaluation Results.](images/evaluate_prompty.png) 112 | 113 |
114 | 115 | ### 02: Conduct a quality evaluation 116 | 117 | This evaluation assesses the overall quality of the generated answers using multiple evaluators provided by Prompt Flow. It considers various dimensions such as **fluency**, **groundedness**, **relevance**, and **coherence**. By evaluating these aspects, we ensure that the answers are not only correct but also well-articulated and contextually appropriate. 118 | 119 | Quality evaluation is ideal for integration into a **continuous integration (CI) pipeline**. By automating this evaluation, you can continuously monitor the AI assistant's performance and catch any regressions or issues early in the development process. Incorporating quality checks into your CI pipeline ensures that each new build meets the predefined quality standards before deployment. 120 | 121 |
122 | Expand this section to view the solution 123 | 124 | To perform the quality evaluation: 125 | 126 | {: .note } 127 | > If steps 1, 2, and 3 have already been executed during the Prompt Evaluation, there is no need to repeat them for the Quality Evaluation. You can proceed directly to running the evaluation script. 128 | 129 | 1. Open a terminal and navigate to the root directory of your project. 130 | 131 | 1. Make sure you have Python installed and the necessary packages by running: 132 | 133 | ```bash 134 | pip install -r requirements.txt 135 | ``` 136 | 137 | 1. Export the **./src** directory to the **PYTHONPATH** to allow Python to find modules in the flow source directory. 138 | 139 | ```bash 140 | export PYTHONPATH=./src:$PYTHONPATH 141 | ``` 142 | 143 | 1. Execute the following evaluation script command: 144 | 145 | ```bash 146 | python evaluations/qa_quality_eval.py 147 | ``` 148 | 149 | This script will perform the quality evaluation using the specified evaluators. 150 | 151 | {: .note } 152 | > I recommend increasing the quota for the gpt-35-turbo model deployment before running the script, so you do not see 429 errors in your terminal output. 153 | 154 | #### Evaluation Results 155 | 156 | *After running the script, you should see output similar to the following:* 157 | 158 | ``` 159 | ... 160 | Check QA evaluation result 241901132937 in the 'Evaluation' section of your project: [AI Project Name]. 161 | ``` 162 | 163 | A file named **qa_flow_quality_eval.json** will be generated, containing the evaluation results. 164 | 165 | ![QA Evaluation Results.](images/evaluate_qa01.png) 166 | 167 | You can also view the results of this evaluation in the Evaluation tab of your project in AI Studio. 168 | 169 | ![QA Evaluation Results in AI Studio.](images/evaluate_qa02.png) 170 | 171 | {: .note } 172 | > There is currently a Bug in AI Foundry so you can't see any of the evaluations on the "Automated evaluations" 173 | 174 |
175 | 176 | ### 03: Perform safety risk evaluation 177 | 178 | This evaluation identifies potential safety risks in the AI's generated responses. Using automated adversarial testing and specialized evaluators, we focus on detecting content related to **sexual content**, **violence**, **self-harm**, **hate speech**, and **unfairness**. This ensures that the AI assistant's responses remain safe and comply with ethical guidelines. 179 | 180 | Safety risk evaluation is crucial during the **testing phase in a controlled environment**. By conducting thorough safety assessments before deploying the AI assistant, you can identify and mitigate potential risks that could lead to harmful or inappropriate outputs. This evaluation helps ensure that the AI system complies with safety standards and regulatory requirements. 181 | 182 | The process assesses how well the AI model maintains safety when exposed to adversarial prompts, including attempts to bypass its safeguards (commonly called "jailbreaks"). An **AdversarialSimulator** generates challenging scenarios designed to provoke the AI into producing prohibited content like sexual material, violent descriptions, or harmful encouragement. The simulation operates in two modes: one without jailbreak attempts and one with jailbreak attempts, generating conversations that test the AI's ability to stick to content policies under pressure. 183 | 184 | After generating these adversarial conversations, the system runs a set of evaluators—**SexualEvaluator**, **ViolenceEvaluator**, **SelfHarmEvaluator**, and **HateUnfairnessEvaluator**—to check if the AI's responses comply with safety guidelines. These evaluators detect any instances of disallowed content in the responses, providing valuable insights into how well the AI maintains safety standards when challenged with harmful inputs. This helps developers refine the model's response mechanisms to ensure safer outputs. 185 | 186 |
187 | Expand this section to view the solution 188 | 189 | To perform the safety risk evaluation: 190 | 191 | {: .note } 192 | > Again, if steps 1, 2, and 3 have already been executed during the previous evaluations, there is no need to repeat them. You can proceed directly to running the evaluation script. 193 | 194 | 1. Open a terminal and navigate to the root directory of your project. 195 | 196 | 1. Make sure you have Python installed and the necessary packages by running: 197 | 198 | ```bash 199 | pip install -r requirements.txt 200 | ``` 201 | 202 | 1. Export the **./src** directory to the **PYTHONPATH** to allow Python to find modules in the flow source directory. 203 | 204 | ```bash 205 | export PYTHONPATH=./src:$PYTHONPATH 206 | ``` 207 | 208 | 4. Execute the following command: 209 | 210 | ```bash 211 | python evaluations/safety_eval.py 212 | ``` 213 | 214 | This script will perform safety evaluations using the specified evaluators. 215 | 216 | {: .note } 217 | > Try re-running the script if you see an error like the following: 218 | 219 | ![Safety eval script error.](images/safety-eval-error.png) 220 | 221 | #### Evaluation Results 222 | 223 | *After running the script, you should see output similar to the following:* 224 | 225 | ``` 226 | ... 227 | Check 241901132937 Adversarial Tests results in the 'Evaluation' section of your project: [AI Project Name]. 228 | ``` 229 | 230 | In the Evaluation section of your AI Studio project, you will see the results table as shown in the figure below. 231 | 232 | ![QA Adversarial Tests Results in AI Studio.](images/evaluate_adversarial01.png) 233 | 234 | Clicking an item shows the Content Safety Assessment results, including a "Metric Dashboard" that categorizes AI-generated responses into Violent, Sexual, Self-harm, and Hateful, with most content rated as "Very low" severity. 235 | 236 | ![QA Adversarial Tests Results in AI Studio.](images/evaluate_adversarial02.png) 237 | 238 |
239 | 240 | **Congratulations on completing this exercise!** 241 | 242 | By following the steps outlined above, you have successfully: 243 | 244 | - Set up the environment and prepared the necessary configurations. 245 | - Performed a prompt-based evaluation to assess the correctness of the AI assistant's responses during. 246 | - Executed a quality evaluation to ensure ongoing high-quality outputs. 247 | - Conducted a safety risk evaluation to identify and address potential safety issues. 248 | 249 | Through these evaluations, you've learned how to implement automated evaluation approaches to enhance scalability and efficiency in monitoring the AI system's performance over time. This comprehensive evaluation strategy ensures that your AI assistant is accurate, high-quality, and safe for deployment. 250 | -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/03_02.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '02: Deploy the RAG flow to an online endpoint' 3 | layout: default 4 | nav_order: 2 5 | parent: 'Exercise 03: Evaluating and Deploying LLMs' 6 | --- 7 | 8 | # Task 02: Deploy the RAG flow to an online endpoint 9 | 10 | ## Description 11 | 12 | In this task, you will use the included reference instructions to deploy the RAG flow to a managed endpoint 13 | 14 | ## Success Criteria 15 | 16 | * Successfully deploy the RAG flow to a remote endpoint 17 | * Verify that the deployment completes successfully 18 | * Verify you can Test the deployment by using the endpoint on the **Test** tab 19 | 20 | ## Solution 21 | 22 | In this task, you will deploy your application to a managed endpoint in Azure by building your flow Docker image, creating an online endpoint, and then creating a deployment in that endpoint. Finally, you will route all traffic to that deployment. 23 | 24 | ### 01: Package your flow as a Docker image 25 | 26 | First, you need to package your flow as a model. This process will create a Dockerfile for your flow. 27 | 28 |
29 | Expand this section to view the solution 30 | 31 | 1. Open a terminal in the root directory of your project. 32 | 33 | 2. Run the following command to build your flow and create a Docker image: 34 | 35 | ```bash 36 | pf flow build --source src --output dist --format docker 37 | ``` 38 | 39 | This command packages your flow and outputs it in the **dist** directory in Docker format. 40 | 41 |
42 | 43 | ### 02: Set the PYTHONPATH environment variable 44 | 45 | To allow Python to find modules in the flow source directory, you need to set the **PYTHONPATH** environment variable. 46 | 47 |
48 | Expand this section to view the solution 49 | 50 | 1. In your terminal, run the following command: 51 | 52 | ```bash 53 | export PYTHONPATH=./src:$PYTHONPATH 54 | ``` 55 | 56 | {: .note } 57 | > Skipping this step will result in a **ModuleNotFoundError: No module named 'chat_request'**. Sometimes you will see errors related to **azure_config** also . This is because the files in /src cannot be found. To test your path, you can type **echo $PYTHONPATH**. You should see **./src**. If you need to modify a corrupted path, you can type **export PYTHONPATH=""** to clear it and try again. 58 | 59 |
60 | 61 | ### 03: Deploy your flow using the deployment script 62 | 63 | Now you are ready to deploy your flow. 64 | 65 |
66 | Expand this section to view the solution 67 | 68 | 1. Open the deployment script: **util/deploy_moe.py** 69 | 70 | 1. Go to **util/deploy_moe.py**, line **77** and update it with the following code to configure your flow to work with the AI Foundry Test Chat interface. 71 | 72 | ``` 73 | model=Model( 74 | name="ragflow", 75 | path=flow_path, # path to promptflow folder 76 | properties=[ # this enables the chat interface in the endpoint test tab 77 | ["azureml.promptflow.source_flow_id", "ragflow"], 78 | ["azureml.promptflow.mode", "chat"], 79 | ["azureml.promptflow.chat_input", "question"], 80 | ["azureml.promptflow.chat_output", "answer"] 81 | ] 82 | ), 83 | ``` 84 | 85 | 1. In **util/deploy_moe.py**, under **environment_variables** on **line 111**, add the following values: 86 | 87 | ``` 88 | "AZURE_TENANT_ID": os.environ["AZURE_TENANT_ID"], 89 | "AZURE_CLIENT_ID": os.environ["AZURE_CLIENT_ID"], 90 | "AZURE_CLIENT_SECRET": os.environ["AZURE_CLIENT_SECRET"], 91 | ``` 92 | 93 | ![deploy script additions](images/deploy-script-additions.png) 94 | 95 | 1. Save changes to **deploy_moe.py**. 96 | 97 | 1. Open the **.env** file in your project folder, replace the following values and add to the file: 98 | 99 | ``` 100 | AZURE_CLIENT_ID=[YOUR_SP_CLIENT_ID] 101 | AZURE_CLIENT_SECRET=[YOUR_SP_CLIENT_SECRET] 102 | ``` 103 | 104 | ![Environment variable additions](images/env-additions.png) 105 | 106 | 1. Save changes to **.env**. 107 | 108 | 1. In the terminal, run the following command. Be sure to replace the placeholder **XXXX** in both the **endpoint-name** and **deployment-name** with a unique four-digit number of your choice. 109 | 110 | ```bash 111 | python util/deploy_moe.py --endpoint-name rag-XXXX-endpoint --deployment-name rag-XXXX-deployment 112 | ``` 113 | 114 | {: .important } 115 | > Both the endpoint and deployment names must be unique within your Azure region. If you see an error indicating that the name is already in use, simply choose different names. Please note, the deployment process can take several minutes to complete. 116 | 117 | {: .note } 118 | > This may take around 20 minutes to deploy. 119 | 120 | 1. Once the deployment completes, you should see output similar to the following in your terminal: 121 | 122 | ![IMAGE OF DEPLOYMENT COMPLETION](images/deployment01.png) 123 | 124 | {: .note} 125 | > If you receive the error "Key based authentication is not permitted on this storage account," enable the option **Allow storage account key access** in the **Configuration** section of your storage account in the Azure portal. 126 | 127 |
128 | 129 | ### 04: Testing the Deployed Flow 130 | 131 | After completing the deployment, you can test it in **AI Studio**. 132 | 133 |
134 | Expand this section to view the solution 135 | 136 | 1. Go to your new deployment in **AI Studio** and open the **Test** tab. 137 | 138 | 1. Ask a question like this: "How can I request a refill for my prescription at Lamna Healthcare?" 139 | 140 | ![IMAGE OF TESTING](images/testdeploy.png) 141 | 142 |
143 | 144 | ### 05: Consuming the Deployed Flow 145 | 146 |
147 | Expand this section to view the solution 148 | 149 | 1. Navigate to your deployment in **AI Studio** and open the **Consume** tab. 150 | 151 | 1. Select the **Python** tab, then copy the example code. 152 | 153 | ![IMAGE OF EXAMPLE CODE](images/deployment04.png) 154 | 155 | 1. Save it as a **test.py** file in the **./local** directory within your repository. 156 | 157 | 1. Before running the program, update the **test.py** file with the **request data** and your **deployment API key** for accessing the deployment. 158 | 159 | ![IMAGE OF WHERE TO MAKE CHANGES](images/deployment05.png) 160 | 161 | Request data: 162 | 163 | ```json 164 | { 165 | "question": "How can I request a refill for my prescription at Lamna Healthcare?", 166 | "chat_history": [] 167 | } 168 | ``` 169 | 170 | 1. Now, you're ready to run the **test.py** program. 171 | 172 | ```bash 173 | python local/test.py 174 | ``` 175 | 176 | ![IMAGE OF PROGRAM EXECUTION](images/deployment06.png) 177 | 178 |
179 | 180 | **Congratulations!** You have successfully deployed your flow to a managed endpoint. You can now integrate this endpoint into your applications or services, allowing you to leverage the power of your RAG (Retrieval-Augmented Generation) flow in a scalable and reliable manner. 181 | -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/03_evaluate_and_deploy_llms.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Exercise 03: Evaluating and Deploying LLMs' 3 | layout: default 4 | nav_order: 4 5 | has_children: true 6 | --- 7 | 8 | # Exercise 03 - Evaluating and Deploying LLMs 9 | 10 | ## Scenario 11 | 12 | In this exercise, you will be using the chat flows within Azure AI Foundry to evaluate and test your large language models (LLMs). This exercise is critical for Lamna Healthcare Company as it aims to create a robust 24/7 support virtual agent that can efficiently manage customer interactions without hallucinations and off topic responses. 13 | 14 | By achieving these objectives, Lamna’s team will enhance their understanding and practical skills in deploying and managing conversational AI solutions using Azure AI Foundry, tailored to their specific healthcare service needs. 15 | 16 | ## Objectives 17 | 18 | After you complete this exercise, you will be able to: 19 | 20 | * Evaluate your Chat flow 21 | * Deploy a RAG flow. 22 | 23 | ## Lab Duration 24 | 25 | * **Estimated Time:** 60 minutes 26 | -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/deploy-script-additions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/deploy-script-additions.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/deployment01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/deployment01.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/deployment02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/deployment02.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/deployment03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/deployment03.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/deployment04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/deployment04.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/deployment05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/deployment05.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/deployment06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/deployment06.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/env-additions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/env-additions.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/evaluate_adversarial01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/evaluate_adversarial01.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/evaluate_adversarial02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/evaluate_adversarial02.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/evaluate_prompty.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/evaluate_prompty.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/evaluate_qa01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/evaluate_qa01.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/evaluate_qa02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/evaluate_qa02.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/safety-eval-error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/safety-eval-error.png -------------------------------------------------------------------------------- /docs/03_evaluate_and_deploy_llms/images/testdeploy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/03_evaluate_and_deploy_llms/images/testdeploy.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/.DS_Store -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/04_01.md: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | title: '01: Monitoring your Prompt Flows' 4 | layout: default 5 | nav_order: 1 6 | parent: 'Exercise 04: Monitoring Prompt Flows' 7 | --- 8 | 9 | # Task 01: Monitoring Your Prompt Flows 10 | 11 | ## Description 12 | 13 | In this task, you will create and configure a monitoring solution to track the quality and token usage of your deployed prompt flow application. 14 | 15 | ## Success Criteria 16 | 17 | - Set up monitoring for Prompt Flow 18 | - Configure monitoring 19 | - Consume monitoring results 20 | 21 | ## Solution 22 | 23 | ### 01: Deploy the RAG flow with tracing enabled 24 | 25 | In this section, you will deploy your flow again, but this time with tracing enabled. 26 | 27 |
28 | Expand this section to view the solution 29 | 30 | 1. Delete the **dist** directory created during the previous deployment 31 | 32 | ```bash 33 | rm -rf dist 34 | ``` 35 | 36 | 1. Package your flow as a Docker image. This process will create a Dockerfile for your flow. 37 | 38 | 1. Open a terminal in the root directory of your project. 39 | 40 | 1. Run the following command to build your flow and create a Docker image: 41 | 42 | ```bash 43 | pf flow build --source src --output dist --format docker 44 | ``` 45 | 46 | This command packages your flow and outputs it to the **dist** directory in Docker format. 47 | 48 | 1. Enable tracing in your flow. 49 | 50 | 1. Open the following file: **dist/flow/flow.flex.yaml** 51 | 52 | 1. Update its content with: 53 | 54 | ``` 55 | app_insights_enabled: true 56 | 57 | inputs: 58 | question: 59 | type: string 60 | chat_history: 61 | type: object 62 | entry: chat_request:get_response 63 | ``` 64 | 65 | 1. Save the file. 66 | 67 | 1. To ensure Python can locate the modules in the flow source directory, you need to set the **PYTHONPATH** environment variable. In your terminal, run the following command: 68 | 69 | ```bash 70 | export PYTHONPATH=./src:$PYTHONPATH 71 | ``` 72 | 73 | {: .note } 74 | > Skipping this step will result in a **ModuleNotFoundError: No module named 'chat_request'**. 75 | 76 | Now, you're ready to deploy your flow. 77 | 78 | 1. Open the deployment script: **util/deploy_moe.py** 79 | 80 | 1. Navigate to line **77** and update it with the following code to configure your flow to work with the AI Foundry Test Chat interface: 81 | 82 | ```python 83 | model = Model( 84 | name="ragwithtrace", 85 | path=flow_path, # path to the promptflow folder 86 | properties=[ # enables the chat interface in the endpoint test tab 87 | ["azureml.promptflow.source_flow_id", "ragwithtrace"], 88 | ["azureml.promptflow.mode", "chat"], 89 | ["azureml.promptflow.chat_input", "question"], 90 | ["azureml.promptflow.chat_output", "answer"] 91 | ] 92 | ), 93 | ``` 94 | 95 | 1. In the terminal, run the following command, replacing **XXXX** in both the **endpoint-name** and **deployment-name** with a unique four-digit number of your choice: 96 | 97 | ```bash 98 | python util/deploy_moe.py --endpoint-name rag-XXXX-endpoint --deployment-name rag-XXXX-deployment 99 | ``` 100 | 101 | {: .important } 102 | > Use a different endpoint and deployment name than the one used in the previous exercise. 103 | 104 | {: .note } 105 | > This may take around 20 minutes to deploy. 106 | 107 | 1. Upon completion, you should see output similar to the following in your terminal: 108 | 109 | ![IMAGE OF DEPLOYMENT COMPLETION](images/monitor01.png) 110 | 111 | {: .note } 112 | > If you encounter the error "Key based authentication is not permitted on this storage account," enable the **Allow storage account key access** option in the **Configuration** section of your storage account in the Azure portal. 113 | 114 |
115 | 116 | ### 02: Create a monitor configuration 117 | 118 |
119 | Expand this section to view the solution 120 | 121 | 1. Create a **monitor.py** file in the **local** folder and add the following content, updating the **Update your Azure resources details** section with your deployment details: 122 | 123 | ```python 124 | from azure.ai.ml import MLClient 125 | from azure.ai.ml.entities import ( 126 | MonitorSchedule, 127 | CronTrigger, 128 | MonitorDefinition, 129 | ServerlessSparkCompute, 130 | MonitoringTarget, 131 | AlertNotification, 132 | GenerationSafetyQualityMonitoringMetricThreshold, 133 | GenerationSafetyQualitySignal, 134 | BaselineDataRange, 135 | LlmData, 136 | ) 137 | from azure.ai.ml.entities._inputs_outputs import Input 138 | from azure.ai.ml.constants import MonitorTargetTasks, MonitorDatasetContext 139 | from azure.identity import DefaultAzureCredential 140 | 141 | credential = DefaultAzureCredential() 142 | 143 | # Update your Azure resources details 144 | subscription_id = "[your_subscription_id]" 145 | resource_group = "[your_resource_group_id]" 146 | aoai_deployment_name = "gpt-4" 147 | aoai_connection_name = "aoai-connection" 148 | project_name = "[your_ai_studio_project_name]" # Ex: ai-project-lh7b37cbhixdq 149 | endpoint_name = "[your_endpoint_name]" # Ex: rag-PCLN-endpoint 150 | deployment_name = "[your_deployment_name]" # Ex: rag-PCLN-deployment 151 | 152 | # These variables can be renamed, but it's not necessary 153 | app_trace_name = "app_traces" 154 | app_trace_version = "1" 155 | monitor_name = "gen_ai_monitor_generation_quality" 156 | defaultgsqsignalname = "gsq-signal" 157 | 158 | # Set the frequency and notification emails for the monitor 159 | trigger_schedule = CronTrigger(expression="*/5 * * * *") 160 | notification_emails_list = ["test@example.com", "def@example.com"] 161 | 162 | ml_client = MLClient( 163 | credential=credential, 164 | subscription_id=subscription_id, 165 | resource_group_name=resource_group, 166 | workspace_name=project_name, 167 | ) 168 | 169 | spark_compute = ServerlessSparkCompute(instance_type="standard_e4s_v3", runtime_version="3.3") 170 | monitoring_target = MonitoringTarget( 171 | ml_task=MonitorTargetTasks.QUESTION_ANSWERING, 172 | endpoint_deployment_id=f"azureml:{endpoint_name}:{deployment_name}", 173 | ) 174 | 175 | # Set thresholds (0.7 = 70%) 176 | aggregated_groundedness_pass_rate = 0.7 177 | aggregated_relevance_pass_rate = 0.7 178 | aggregated_coherence_pass_rate = 0.7 179 | aggregated_fluency_pass_rate = 0.7 180 | 181 | # Create a Generation Safety Quality (GSQ) signal 182 | generation_quality_thresholds = GenerationSafetyQualityMonitoringMetricThreshold( 183 | groundedness={"aggregated_groundedness_pass_rate": aggregated_groundedness_pass_rate}, 184 | relevance={"aggregated_relevance_pass_rate": aggregated_relevance_pass_rate}, 185 | coherence={"aggregated_coherence_pass_rate": aggregated_coherence_pass_rate}, 186 | fluency={"aggregated_fluency_pass_rate": aggregated_fluency_pass_rate}, 187 | ) 188 | input_data = Input( 189 | type="uri_folder", 190 | path=f"{endpoint_name}-{deployment_name}-{app_trace_name}:{app_trace_version}", 191 | ) 192 | data_window = BaselineDataRange(lookback_window_size="P7D", lookback_window_offset="P0D") 193 | production_data = LlmData( 194 | data_column_names={"prompt_column": "question", "completion_column": "answer", "context_column": "context"}, 195 | input_data=input_data, 196 | data_window=data_window, 197 | ) 198 | 199 | gsq_signal = GenerationSafetyQualitySignal( 200 | connection_id=f"/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/Microsoft.MachineLearningServices/workspaces/{project_name}/connections/{aoai_connection_name}", 201 | metric_thresholds=generation_quality_thresholds, 202 | production_data=[production_data], 203 | sampling_rate=1.0, 204 | properties={ 205 | "aoai_deployment_name": aoai_deployment_name, 206 | "enable_action_analyzer": "false", 207 | "azureml.modelmonitor.gsq_thresholds": '[{"metricName":"average_fluency","threshold":{"value":4}},{"metricName":"average_coherence","threshold":{"value":4}}]', 208 | }, 209 | ) 210 | 211 | monitoring_signals = { 212 | defaultgsqsignalname: gsq_signal, 213 | } 214 | 215 | monitor_settings = MonitorDefinition( 216 | compute=spark_compute, 217 | monitoring_target=monitoring_target, 218 | monitoring_signals=monitoring_signals, 219 | alert_notification=AlertNotification(emails=notification_emails_list), 220 | ) 221 | 222 | model_monitor = MonitorSchedule( 223 | name=monitor_name, 224 | trigger=trigger_schedule, 225 | create_monitor=monitor_settings, 226 | ) 227 | 228 | ml_client.schedules.begin_create_or_update(model_monitor) 229 | ``` 230 | 231 | 1. Now, run the **monitor.py** program to create your monitor configuration: 232 | 233 | ```bash 234 | python local/monitor.py 235 | ``` 236 | 237 | ![MONITORING](images/monitor02.png) 238 | 239 | Your monitoring configuration is now set up. 240 | 241 |
242 | 243 | ### 03: Send some requests to your endpoint to generate monitoring data 244 | 245 |
246 | Expand this section to view the solution 247 | 248 | 1. Update the **local/test.py** program created earlier with the new endpoint and API key 249 | 250 | ![MONITORING](images/monitor03.png) 251 | 252 | 1. Save the file. 253 | 254 | 1. Run the **local/test.py** program multiple times to generate some requests 255 | 256 | ```bash 257 | python local/test.py 258 | ``` 259 | 260 | ![MONITORING](images/monitor04.png) 261 | 262 | {: .note } 263 | > You can vary the question content for each request by modifying it on line **19** of the **test.py** program. 264 | 265 |
266 | 267 | ### 04: Consume monitoring results 268 | 269 | Once you've set up your monitor, it will run accordingly the schedule you set in the monitoring configuration. 270 | 271 |
272 | Expand this section to view the solution 273 | 274 | 1. Go to the **Monitoring (preview)** tab within the deployment to view the monitoring results. 275 | 276 | 1. Go to the **Generation quality** tab to monitor the quality of your application over time. Metrics shown include: 277 | 278 | - **Violation count**: Sum of violations for a given metric (e.g., Fluency) during the time window. 279 | - **Average score**: Average score for all instances (or requests) over the time window. 280 | 281 | The **Generation quality violations** card shows the **violation rate**, which is the number of violations divided by the total number of possible violations. Adjust thresholds and computation frequency (default: daily) in the settings. 282 | 283 | ![LLMOps Workshop](images/dash_quality.png) 284 | 285 | 1. Go to the **Operational** tab to view near real-time operational metrics for the deployment, including: 286 | 287 | - Request count 288 | - Latency 289 | - Error rate 290 | 291 | ![LLMOps Workshop](images/lab4grab12.png) 292 | 293 | [Reference](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/monitor-quality-safety?tabs=azure-studio#advanced-monitoring-configuration-with-sdk-v2) 294 | 295 | ### 05: Enable tracing in Application Insights 296 | 297 | You can enable tracing in Application Insights by selecting your workspace's Application Insights. 298 | 299 |
300 | Expand this section to view the solution 301 | 302 | 1. Open Application Insights, select **Usage and estimated costs** under **Configure** from the left menu. 303 | 304 | 1. Select **Custom metrics (Preview)**. 305 | 306 | 1. Select **With dimensions**, then select **OK** to save changes. 307 | 308 | 1. Select **Metrics** under **Monitoring** from the left menu. 309 | 310 | 1. Select **Promptflow standard metrics** from the Metric Namespace, and explore metrics using different aggregation methods. 311 | 312 | 1. Select **Transaction search** from the left menu to view tracing in the transactional data. 313 | 314 | ![LLMOps Workshop](images/dash_insights.png) 315 | 316 | [Reference](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/trace-production-sdk) 317 | 318 |
319 | -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/04_monitoring_llms.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Exercise 04: Monitoring Prompt Flows' 3 | layout: default 4 | nav_order: 5 5 | has_children: true 6 | --- 7 | 8 | # Exercise 04 - Monitoring LLM's 9 | 10 | ## Scenario 11 | 12 | In this exercise, you will be adding monitoring capabilities to your LLM flows . This is essential for Lamna Healthcare Company to ensure the optimal performance and reliability of their 24/7 support virtual agent. 13 | 14 | By completing this exercise, Lamna’s team will be equipped with the tools and knowledge to maintain a high standard of service quality and safety in their AI-driven customer support system. 15 | 16 | ## Objectives 17 | 18 | After you complete this exercise, you will be able to: 19 | 20 | * Monitoring your LLMs flows 21 | 22 | ## Lab Duration 23 | 24 | * **Estimated Time:** 60 minutes 25 | -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/draft backup/Exercise04.md: -------------------------------------------------------------------------------- 1 | #### Monitoring 2 | 3 | #### Prerequisites 4 | 5 | An Azure subscription where you can create an AI Hub Resource and a AI Search service. 6 | 7 | #### Setup 8 | 9 | If you are running this exercise after lesson 1, you don't need to worry about this step. 10 | 11 | Otherwise, follow **Setup** from **Lesson 1** to create a project and its associated resources in Azure AI Studio, as well as to deploy the GPT-4 model. 12 | 13 | #### Exercise Steps 14 | 15 | In this exercise, you will execute the following steps: 16 | 17 | 1) Monitoring your LLMs flow. 18 | 19 | 2) Add Content Safety to your Solution. 20 | 21 | ##### 1) Monitoring your LLMs flow 22 | 23 | Modify the output node of the workflow to incorporate the required information for computing the metrics that need monitoring, as outlined in the [User Experience](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/monitor-quality-safety#user-experience) section of the Monitor Quality and Safety of Deployed Applications documentation. Be sure to activate monitoring by selecting the "Enable" button within the Model Monitoring section when deploying the workflow. 24 | 25 | ##### 2) Add Content Safety to your Solution 26 | 27 | Create a basic **flow** from scratch (a simple Joke flow will be constructed). 28 | 29 | Make the following adjustments to the flow as per the instructions below: 30 | 31 | - Insert a [Content Safety tool](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/tools-reference/content-safety-text-tool) node between the input and the language model (LLM). 32 | 33 | - Add a Python node to process the output from the Content Safety tool and determine whether to proceed with the standard flow or not. For guidance on creating a conditional flow in Prompt Flow, refer to [this example](https://github.com/microsoft/promptflow/tree/main/examples/flows/standard/conditional-flow-for-if-else). 34 | 35 | - Add a Python node to craft a default response. 36 | 37 | - Implement conditions to trigger one response or another, depending on the Content Safety's result. 38 | 39 | - Develop a final node to compile and deliver the response to the user. -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/dash_endpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/dash_endpoint.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/dash_insights.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/dash_insights.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/dash_operational.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/dash_operational.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/dash_quality.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/dash_quality.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab1.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab10.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab11.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab12.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab13.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab14.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab15.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab16.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab17.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab18.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab19.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab2.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab20.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab21.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab22.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab22.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab23.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab23.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab24.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab24.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab3.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab4.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab5.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab6.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab7.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab8.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/lab4grab9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/lab4grab9.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/monitor01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/monitor01.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/monitor02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/monitor02.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/monitor03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/monitor03.png -------------------------------------------------------------------------------- /docs/04_monitoring_llm_flows/images/monitor04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/04_monitoring_llm_flows/images/monitor04.png -------------------------------------------------------------------------------- /docs/05_automating_everything/05_01.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '01: Delivering New Feature' 3 | layout: default 4 | nav_order: 1 5 | parent: 'Exercise 05: Automate Everything' 6 | --- 7 | 8 | # Task 01: Delivering a New Feature 9 | 10 | Once the project bootstrapping is complete, the team can begin developing new features. This section provides a detailed guide on delivering a new feature, covering every step from initial development to production deployment. To illustrate the procedure, we will develop a new feature called "Feature X," which will be included in the project's release 1.0.0. The process can be summarized in six steps, represented in the following diagram, making it easier to understand and follow along. 11 | 12 | ![Git Workflow](images/git_workflow_branching.png) 13 | 14 | ## Solution 15 | 16 | Follow the steps below to deliver this feature from the beginning of development to deployment in production. You will need access to your project repository that has been bootstrapped, a terminal (bash or PowerShell) with Git, and the GitHub page of your repository. 17 | 18 | {: .important } 19 | > For the correct execution of all github action Pipelines we need to do a few fixes on the github actions workflows 20 | > For All yaml files (pr_pipelines.yml, continuous_integration.yml, continuous_delivery_qa.yml, continuous_delivery_prod.yml, continuous_delivery_dev.yml) 21 | > Change the azd setup version from Azure/setup-azd@v1.0.0 to Azure/setup-azd@v2.1.0 22 | > Change the python version from python-version: 3.8 to python-version: 3.11 23 | 24 | 25 | {: .important } 26 | > On the file ./.github/actions/config-env/action.yml on the Provision environment step, on the Previous deployment exists section (line 231) before the azd provision line, add a line azd env set AZURE_PRINCIPAL_TYPE 'ServicePrincipal' 27 | 28 | {: .important } 29 | > On the file ./infra/main.bicep comment out all roles provisioning, from line 144 to 275 30 | 31 | {: .important } 32 | > On the file ./evaluations/qa_quality_eval.py add a check if the file exist for data, and a try/except block for the base run, response and convert to jsonl 33 | 34 | {: .important } 35 | > On the file ./evaluations/safety_eval.py add a try/except block for simulator, adversarial_eval_result, and adversarial_eval_w_jailbreak_result 36 | 37 | {: .important } 38 | > If a deployment fails, the next time we retry the deployment azd wont be able to refresh the env variables correctly, to fix this go to the Azure Portal>Subscription>Deployments and remove the failed deployment for the environment 39 | 40 | ### 01: Create a feature branch 41 | 42 | {: .note } 43 | > If you completed Exercise 02: Building LLMs Orchestration Flows, you can skip to the next section. 44 | 45 | The workflow starts by creating a feature branch named **feature/feature_x** from the **develop** branch. This is where the development team will work on the new feature x. 46 | 47 |
48 | Expand this section to view the solution 49 | 50 | 1. Switch to the **develop** branch and pull the latest changes: 51 | 52 | ```bash 53 | git checkout develop 54 | git pull 55 | ``` 56 | {: .note } 57 | > If you do not have a **develop** branch you can use Git commands to create one. Ensure that you have created all the Environment variables in Prod, QA, Dev within your GitHub repo. Also ensure that you have added the SP secrets to each. 58 | 59 | 1. Create the feature branch: 60 | 61 | ```bash 62 | git checkout -b feature/feature_x 63 | ``` 64 | 65 | 1. Make non-disruptive changes to the repository. For instance, create a file **FEATUREX.md** in the project root: 66 | 67 | *Using Bash:* 68 | 69 | ```bash 70 | touch FEATUREX.md 71 | ``` 72 | 73 | *Using PowerShell:* 74 | 75 | ```powershell 76 | New-Item -ItemType File -Name "FEATUREX.md" 77 | ``` 78 | 79 | This ensures the new feature is developed in isolation, maintaining the integrity of the project's **develop** branch and promptflow. 80 | 81 |
82 | 83 | ### 02: Pull Request (PR) to `develop` 84 | 85 | Upon completing the feature, create a Pull Request (PR) to merge changes from the feature branch **feature/feature_x** to the **develop** branch, which is the default branch where the team integrates changes. 86 | 87 |
88 | Expand this section to view the solution 89 | 90 | 1. Add changes, commit, and push to the feature branch: 91 | 92 | ```bash 93 | git add . 94 | git commit -m "Feature X complete" 95 | git push origin feature/feature_x 96 | ``` 97 | 98 | 1. Create the PR: 99 | 100 | ```bash 101 | gh pr create --base develop --head feature/feature_x --title "Feature X" --body "Description of the changes and the impact." 102 | ``` 103 | 104 | {: .note } 105 | > If needed, install **gh** using **sudo apt install gh**, then login with **gh auth login**. 106 | 107 | You can also use the GitHub website to create the pull request. Remember to select **develop** as the base branch and **feature/feature_x** as the compare branch. 108 | 109 | The creation of the PR triggers a PR Evaluation Pipeline to ensure that the code adheres to standards, passes unit tests, and the orchestration flow is evaluated by AI to ensure it meets quality metrics. 110 | 111 |
112 | 113 | ### 03: Merge to `develop` 114 | 115 | Approve the Pull Request, merging it into the **develop** branch. This merge triggers the Continuous Integration (CI) Pipeline, which builds the orchestration flow and conducts AI-assisted evaluations using a comprehensive test dataset based on the [Golden Dataset](https://aka.ms/copilot-golden-dataset-guide). Upon successful completion, the Continuous Deployment (CD) Pipeline is executed to deploy the flow to the **dev** environment. 116 | 117 |
118 | Expand this section to view the solution 119 | 120 | 1. Merge the PR using GitHub by going to the **Pull Requests** tab in your repository, select the recently created PR, and select **Merge pull request**. 121 | 122 |
123 | 124 | ## Optional Steps 125 | The following steps are optional and can be skipped if you prefer to conclude the exercise after merging to develop. These steps are recommended for additional testing and quality assurance before deploying to production. 126 | 127 | ### 04. Release Branch (`release/1.0.0`) **(Optional)** 128 | 129 | {: .note } 130 | > Creating a release branch is not mandatory but recommended for additional testing and quality assurance before deploying to production. Proceed with this step if your workflow benefits from a dedicated release phase. 131 | 132 | After confirming the stability of the **develop**`branch through testing in **dev**, create a release branch **release/1.0.0** from **develop**. This triggers a *Continuous Deployment (CD) pipeline* to deploy the application to the **qa** environment. Before deployment, an AI-based evaluation assesses [quality](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/flow-evaluate-sdk), risk, and [safety](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/simulator-interaction-data) evaluations. The application in **qa** is then used for User Acceptance Testing (UAT) and [red-teaming](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/red-teaming) or LLM App. 133 | 134 |
135 | Expand this section to view the solution 136 | 137 | 1. Create the release branch: 138 | 139 | ```bash 140 | git checkout develop 141 | git pull origin develop 142 | git checkout -b release/1.0.0 143 | git push origin release/1.0.0 144 | ``` 145 | 146 |
147 | 148 | ### 05: Pull Request to `main` 149 | 150 | After UAT tests in the **qa** environment confirm that the application is ready for production, create a Pull Request (PR) to merge the changes into the **main** branch from the **release/1.0.0** branch. 151 | 152 |
153 | Expand this section to view the solution 154 | 155 | 1. Create the PR: 156 | 157 | Below is an example utilizing the GitHub CLI: 158 | 159 | ```bash 160 | gh pr create --base main --head release/1.0.0 --title "Release 1.0.0" --body "Merging release/1.0.0 into main after successful UAT in QA environment" 161 | ``` 162 | 163 | You can also use the GitHub website to create the pull request. Remember to select **main** as the base branch and **release/1.0.0** as the compare branch. 164 | 165 |
166 | 167 | ### 06: Merge to `main` 168 | 169 |
170 | Expand this section to view the solution 171 | 172 | 1. Once the Pull Request (PR) to the **main** branch is approved on GitHub, go to the **Pull Requests** tab of your project repository on GitHub. 173 | 174 | 1. Select the PR created for merging into production, and select **Merge pull request** to manually approve the merge of **release/1.0.0** into the **main** branch. This action triggers the Continuous Deployment (CD) pipeline, which deploys the code to the **prod** environment. 175 | 176 |
177 | -------------------------------------------------------------------------------- /docs/05_automating_everything/05_automating_everything.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Exercise 05: Automate Everything' 3 | layout: default 4 | nav_order: 6 5 | has_children: true 6 | --- 7 | 8 | # Exercise 05: Automating Everything 9 | 10 | ## Scenario 11 | 12 | In this exercise, you'll learn to automate the build, evaluation, and deployment of your LLM orchestration flow. We'll use the [**GenAIOps Accelerator**](https://github.com/azure/genaiops) as a guide for deploying an LLM with **Prompt Flow** and **GitHub Actions**. This automation is vital for Lamna Healthcare Company, enabling efficient, error-free deployment processes for their 24/7 support virtual agent. 13 | 14 | By mastering these tasks, Lamna Healthcare Company's team will boost their ability to manage AI projects independently, ensuring smooth operations and maximizing the benefits of Azure AI services for their healthcare solutions. 15 | 16 | ## Git Workflow 17 | 18 | The image below illustrates the workflow used in the workshop. We'll explore how to deliver a new feature based on this workflow. In this example, we are developing a feature called "Feature X," which will be included in the project's release 1.0.0. 19 | 20 | ![Git Workflow](images/git_workflow_branching.png) 21 | 22 | ## Detailed Workflow Description: 23 | 24 | 1. **Feature Branch Creation:** 25 | 26 | The process starts when the development team creates a feature branch from the `develop` branch. This branch is dedicated to the development of the new feature X. 27 | 28 | 2. **Pull Request (PR):** 29 | 30 | Upon completing the feature, a Pull Request (PR) is initiated from the feature branch to the `develop` branch, which is the default branch where the team integrates changes. 31 | 32 | The creation of the PR triggers a *PR Evaluation Pipeline* to ensure that the code adheres to standards, passes unit tests, and the orchestration flow is evaluated by AI to ensure it meets quality metrics. 33 | 34 | 3. **Merge to develop:** 35 | 36 | Once the Pull Request is approved, it is merged into the `develop` branch. This merge triggers the *Continuous Integration (CI) Pipeline*, which builds the orchestration flow and conducts AI-assisted evaluations using a comprehensive test dataset based on the [Golden Dataset](https://aka.ms/copilot-golden-dataset-guide). Upon successful completion, the *Continuous Deployment (CD) Pipeline* is executed to deploy the flow to the **dev** environment. 37 | 38 | 4. **Release Branch Creation (Release/1.0.0):** 39 | 40 | After confirming the stability of the `develop` branch through testing in **dev**, a release branch is created from `develop`. This triggers a *Continuous Deployment (CD) Pipeline* to deploy the application to the **qa** environment. Before deployment, an AI-based evaluation assesses [quality](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/flow-evaluate-sdk), risk, and [safety](https://learn.microsoft.com/en-us/azure/ai-studio/how-to/develop/simulator-interaction-data) factors. The application in **qa** is then used for User Acceptance Testing (UAT) and [red-teaming](https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/red-teaming) or LLM App. 41 | 42 | 5. **Pull Request to main:** 43 | 44 | After UAT tests in the **qa** environment confirm that the application is ready for production, a Pull Request (PR) is created to merge the changes into the `main` branch. 45 | 46 | 6. **Merge to main:** 47 | 48 | Once the Pull Request (PR) to the `main` branch is manually approved, the release branch is merged into the `main` branch. This action triggers the Continuous Deployment (CD) Pipeline, which deploys the code to the **prod** environment. 49 | 50 | ## CI/CD Pipelines 51 | 52 | The CI/CD (Continuous Integration/Continuous Deployment) pipelines automate integration, evaluation, and deployment processes, ensuring efficient delivery of high-quality applications. 53 | 54 | ![Pipelines](images/git_workflow_pipelines.png) 55 | 56 | - **The Pull Request Evaluation Pipeline** begins with unit tests, followed by a code review, and concludes with AI-assisted prompt evaluation to validate code changes before integration. 57 | 58 | - **The Continuous Integration Pipeline** starts with unit tests and code reviews, followed by AI-assisted flow evaluation to identify potential issues. The application is then built, and the flow image is registered for deployment. 59 | 60 | - **The Continuous Deployment Pipeline** operates across three environments: dev, qa, and prod. Provisioning of resources is performed when necessary, and the deployment of the application is executed in the respective environment. 61 | 62 | - **In the dev environment**, the latest code is pulled, and the application is deployed for the development team's testing. 63 | 64 | - **In the qa environment**, the code is retrieved, and AI-assisted evaluations for quality and safety are conducted, followed by integration testing. The application is then deployed and made available for User Acceptance Testing (UAT). 65 | 66 | - **In the prod environment**, the same image built in the Continuous Integration Pipeline is deployed, ensuring consistency and reliability. Integration testing is conducted, and smoke testing ensures functionality post-deployment. 67 | 68 | This structured approach streamlines workflows, reduces errors, and guarantees the efficient delivery of applications to production. 69 | 70 | ## Objectives 71 | 72 | After you complete this exercise, you will be able to: 73 | 74 | - Discuss further the topics of this exercise 75 | - Create a project and associate resources without instruction 76 | 77 | ## Lab Duration 78 | 79 | - **Estimated Time:** 120 minutes -------------------------------------------------------------------------------- /docs/05_automating_everything/images/git_workflow_branching.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/git_workflow_branching.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/git_workflow_pipelines.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/git_workflow_pipelines.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab1.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab10.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab11.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab12.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab13.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab14.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab15.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab16.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab17.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab18.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab2.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab3.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab4.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab4b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab4b.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab4c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab4c.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab5.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab6.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab7.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab8.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/lab6grab9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/lab6grab9.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/large-language-model-operations-prompt-flow-process.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/large-language-model-operations-prompt-flow-process.png -------------------------------------------------------------------------------- /docs/05_automating_everything/images/llmops-with-prompt-flow-diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/TechExcel-Operationalize-LLMs-with-LLMOps-automation/17dc849e3c0277e6d882186db61d5a9f9668f327/docs/05_automating_everything/images/llmops-with-prompt-flow-diagram.png -------------------------------------------------------------------------------- /index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Introduction 3 | layout: home 4 | nav_order: 1 5 | --- 6 | 7 | # TechExcel: Operationalize LLMs with GenAIOps automation 8 | 9 | ## Azure services and related products 10 | 11 | - Azure AI Foundry 12 | - Azure OpenAI 13 | - Azure Hub 14 | - Azure Prompt Flows 15 | - Azure AI Deployments 16 | - Azure Content Safety 17 | 18 | ## Solution architecture 19 | 20 | ![Diagram showing Azure AI Studio and the scope of use within the environment](resource-provider-connected-resources.svg) 21 | 22 | The diagram shows the solution architecture of an AI project within Azure AI Foundry. The AI Foundry Hub includes the Azure Subcription, Resource Group, and Azure Content Safety service. The AI Foundry Hub provides integration with services including Azure OpenAI, Azure AI Search, and Azure Storage Accounts to implement AI-based solutions. The AI Playground offers tools like Chat Playground, Deployments, and Prompt Flow, that enable quick and easy access to AI models such as GPT-4 and Llama-2 and interactive development tools. 23 | 24 | ## Exercises 25 | 26 | These labs have exercises on: 27 | 28 | * Configuring and setting up Azure AI Foundry Project 29 | * Set up a Conversational RAG Flow 30 | * Emulated chat interactions and testing 31 | * Creating and testing multiple prompt flows and models 32 | * Using a notebook to setup and configure a project 33 | 34 | These lab are available as GitHub pages [here](https://microsoft.github.io/TechExcel-Operationalize-LLMs-with-LLMOps-automation) 35 | 36 | ## Prerequisites 37 | 38 | An Azure external subscription is required, where you can create an AI Project along with its AI Hub Resource, a Content Safety service, a new instance of Azure OpenAI, and an AI Search service. 39 | --------------------------------------------------------------------------------