├── .github ├── CODE_OF_CONDUCT.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Demos ├── .env ├── Extensions │ ├── AI-Toolkit-Extension │ │ ├── Assets │ │ │ ├── MCP_command.txt │ │ │ ├── brand-style-guide.docx │ │ │ ├── cta-evaluation-data.jsonl │ │ │ ├── cta-evaluator.md │ │ │ └── single-post-schema.json │ │ ├── demo-aitk-create-agent-mcp-tools.md │ │ ├── demo-aitk-evaluate-model-responses.md │ │ ├── demo-aitk-explore-compare-models.md │ │ └── requirements.txt │ └── Azure-AI-Foundry-Extension │ │ ├── Assets │ │ ├── bing_grounding.yml │ │ └── system-prompt.md │ │ ├── demo-aifx-create-agent-tools.md │ │ ├── demo-aifx-explore-compare-models.md │ │ └── requirements.txt └── README.md ├── LICENSE.md ├── README.md ├── SECURITY.md └── Samples ├── 1. Basic ├── AI_Toolkit_Streamlit.py ├── AI_Toolkit_basic.py └── AI_Toolkit_console.py ├── 2. RAG ├── Embeddings │ ├── embeddings.csv │ ├── embeddings.py │ └── paragraph.txt ├── RAG_GithubModel │ ├── ai-toolkit │ │ ├── 2f116a22-ece6-433b-9427-efbb2af4f4e2 │ │ │ ├── data_level0.bin │ │ │ ├── header.bin │ │ │ ├── length.bin │ │ │ └── link_lists.bin │ │ └── chroma.sqlite3 │ ├── cloud_embedding_model.ipynb │ ├── cloud_model_rag.py │ └── docs │ │ └── AIToolkit.pdf └── Streamlit_PDF_OfflineModel │ ├── Console.py │ ├── ai-toolkit │ ├── bafc4831-b7f7-461f-a499-08a5a0245047 │ │ ├── data_level0.bin │ │ ├── header.bin │ │ ├── length.bin │ │ └── link_lists.bin │ └── chroma.sqlite3 │ ├── app.py │ ├── dbmaker.ipynb │ ├── docs │ ├── AIToolkit.pdf │ └── Finetune.pdf │ └── requirements.txt └── 3.Multimodal ├── Basic Console ├── app.py └── img.jpg ├── Recipe Generator └── app.py └── requirements.txt /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | > Please provide us with the following information: 5 | > --------------------------------------------------------------- 6 | 7 | ### This issue is for a: (mark with an `x`) 8 | ``` 9 | - [ ] bug report -> please search issues before submitting 10 | - [ ] feature request 11 | - [ ] documentation issue or request 12 | - [ ] regression (a behavior that used to work and stopped in a new release) 13 | ``` 14 | 15 | ### Minimal steps to reproduce 16 | > 17 | 18 | ### Any log messages given by the failure 19 | > 20 | 21 | ### Expected/desired behavior 22 | > 23 | 24 | ### OS and Version? 25 | > Windows 7, 8 or 10. Linux (which distribution). macOS (Yosemite? El Capitan? Sierra?) 26 | 27 | ### Versions 28 | > 29 | 30 | ### Mention any other details that might be useful 31 | 32 | > --------------------------------------------------------------- 33 | > Thanks! We'll be in touch soon. 34 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | 3 | * ... 4 | 5 | ## Does this introduce a breaking change? 6 | 7 | ``` 8 | [ ] Yes 9 | [ ] No 10 | ``` 11 | 12 | ## Pull Request Type 13 | What kind of change does this Pull Request introduce? 14 | 15 | 16 | ``` 17 | [ ] Bugfix 18 | [ ] Feature 19 | [ ] Code style update (formatting, local variables) 20 | [ ] Refactoring (no functional changes, no api changes) 21 | [ ] Documentation content changes 22 | [ ] Other... Please describe: 23 | ``` 24 | 25 | ## How to Test 26 | * Get the code 27 | 28 | ``` 29 | git clone [repo-address] 30 | cd [repo-name] 31 | git checkout [branch-name] 32 | npm install 33 | ``` 34 | 35 | * Test the code 36 | 37 | ``` 38 | ``` 39 | 40 | ## What to Check 41 | Verify that the following are valid 42 | * ... 43 | 44 | ## Other Information 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore 5 | 6 | # User-specific files 7 | *.rsuser 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Mono auto generated files 17 | mono_crash.* 18 | 19 | # Build results 20 | [Dd]ebug/ 21 | [Dd]ebugPublic/ 22 | [Rr]elease/ 23 | [Rr]eleases/ 24 | x64/ 25 | x86/ 26 | [Ww][Ii][Nn]32/ 27 | [Aa][Rr][Mm]/ 28 | [Aa][Rr][Mm]64/ 29 | bld/ 30 | [Bb]in/ 31 | [Oo]bj/ 32 | [Ll]og/ 33 | [Ll]ogs/ 34 | 35 | # Visual Studio 2015/2017 cache/options directory 36 | .vs/ 37 | # Uncomment if you have tasks that create the project's static files in wwwroot 38 | #wwwroot/ 39 | 40 | # Visual Studio 2017 auto generated files 41 | Generated\ Files/ 42 | 43 | # MSTest test Results 44 | [Tt]est[Rr]esult*/ 45 | [Bb]uild[Ll]og.* 46 | 47 | # NUnit 48 | *.VisualState.xml 49 | TestResult.xml 50 | nunit-*.xml 51 | 52 | # Build Results of an ATL Project 53 | [Dd]ebugPS/ 54 | [Rr]eleasePS/ 55 | dlldata.c 56 | 57 | # Benchmark Results 58 | BenchmarkDotNet.Artifacts/ 59 | 60 | # .NET Core 61 | project.lock.json 62 | project.fragment.lock.json 63 | artifacts/ 64 | 65 | # ASP.NET Scaffolding 66 | ScaffoldingReadMe.txt 67 | 68 | # StyleCop 69 | StyleCopReport.xml 70 | 71 | # Files built by Visual Studio 72 | *_i.c 73 | *_p.c 74 | *_h.h 75 | *.ilk 76 | *.meta 77 | *.obj 78 | *.iobj 79 | *.pch 80 | *.pdb 81 | *.ipdb 82 | *.pgc 83 | *.pgd 84 | *.rsp 85 | *.sbr 86 | *.tlb 87 | *.tli 88 | *.tlh 89 | *.tmp 90 | *.tmp_proj 91 | *_wpftmp.csproj 92 | *.log 93 | *.tlog 94 | *.vspscc 95 | *.vssscc 96 | .builds 97 | *.pidb 98 | *.svclog 99 | *.scc 100 | 101 | # Chutzpah Test files 102 | _Chutzpah* 103 | 104 | # Visual C++ cache files 105 | ipch/ 106 | *.aps 107 | *.ncb 108 | *.opendb 109 | *.opensdf 110 | *.sdf 111 | *.cachefile 112 | *.VC.db 113 | *.VC.VC.opendb 114 | 115 | # Visual Studio profiler 116 | *.psess 117 | *.vsp 118 | *.vspx 119 | *.sap 120 | 121 | # Visual Studio Trace Files 122 | *.e2e 123 | 124 | # TFS 2012 Local Workspace 125 | $tf/ 126 | 127 | # Guidance Automation Toolkit 128 | *.gpState 129 | 130 | # ReSharper is a .NET coding add-in 131 | _ReSharper*/ 132 | *.[Rr]e[Ss]harper 133 | *.DotSettings.user 134 | 135 | # TeamCity is a build add-in 136 | _TeamCity* 137 | 138 | # DotCover is a Code Coverage Tool 139 | *.dotCover 140 | 141 | # AxoCover is a Code Coverage Tool 142 | .axoCover/* 143 | !.axoCover/settings.json 144 | 145 | # Coverlet is a free, cross platform Code Coverage Tool 146 | coverage*.json 147 | coverage*.xml 148 | coverage*.info 149 | 150 | # Visual Studio code coverage results 151 | *.coverage 152 | *.coveragexml 153 | 154 | # NCrunch 155 | _NCrunch_* 156 | .*crunch*.local.xml 157 | nCrunchTemp_* 158 | 159 | # MightyMoose 160 | *.mm.* 161 | AutoTest.Net/ 162 | 163 | # Web workbench (sass) 164 | .sass-cache/ 165 | 166 | # Installshield output folder 167 | [Ee]xpress/ 168 | 169 | # DocProject is a documentation generator add-in 170 | DocProject/buildhelp/ 171 | DocProject/Help/*.HxT 172 | DocProject/Help/*.HxC 173 | DocProject/Help/*.hhc 174 | DocProject/Help/*.hhk 175 | DocProject/Help/*.hhp 176 | DocProject/Help/Html2 177 | DocProject/Help/html 178 | 179 | # Click-Once directory 180 | publish/ 181 | 182 | # Publish Web Output 183 | *.[Pp]ublish.xml 184 | *.azurePubxml 185 | # Note: Comment the next line if you want to checkin your web deploy settings, 186 | # but database connection strings (with potential passwords) will be unencrypted 187 | *.pubxml 188 | *.publishproj 189 | 190 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 191 | # checkin your Azure Web App publish settings, but sensitive information contained 192 | # in these scripts will be unencrypted 193 | PublishScripts/ 194 | 195 | # NuGet Packages 196 | *.nupkg 197 | # NuGet Symbol Packages 198 | *.snupkg 199 | # The packages folder can be ignored because of Package Restore 200 | **/[Pp]ackages/* 201 | # except build/, which is used as an MSBuild target. 202 | !**/[Pp]ackages/build/ 203 | # Uncomment if necessary however generally it will be regenerated when needed 204 | #!**/[Pp]ackages/repositories.config 205 | # NuGet v3's project.json files produces more ignorable files 206 | *.nuget.props 207 | *.nuget.targets 208 | 209 | # Microsoft Azure Build Output 210 | csx/ 211 | *.build.csdef 212 | 213 | # Microsoft Azure Emulator 214 | ecf/ 215 | rcf/ 216 | 217 | # Windows Store app package directories and files 218 | AppPackages/ 219 | BundleArtifacts/ 220 | Package.StoreAssociation.xml 221 | _pkginfo.txt 222 | *.appx 223 | *.appxbundle 224 | *.appxupload 225 | 226 | # Visual Studio cache files 227 | # files ending in .cache can be ignored 228 | *.[Cc]ache 229 | # but keep track of directories ending in .cache 230 | !?*.[Cc]ache/ 231 | 232 | # Others 233 | ClientBin/ 234 | ~$* 235 | *~ 236 | *.dbmdl 237 | *.dbproj.schemaview 238 | *.jfm 239 | *.pfx 240 | *.publishsettings 241 | orleans.codegen.cs 242 | 243 | # Including strong name files can present a security risk 244 | # (https://github.com/github/gitignore/pull/2483#issue-259490424) 245 | #*.snk 246 | 247 | # Since there are multiple workflows, uncomment next line to ignore bower_components 248 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 249 | #bower_components/ 250 | 251 | # RIA/Silverlight projects 252 | Generated_Code/ 253 | 254 | # Backup & report files from converting an old project file 255 | # to a newer Visual Studio version. Backup files are not needed, 256 | # because we have git ;-) 257 | _UpgradeReport_Files/ 258 | Backup*/ 259 | UpgradeLog*.XML 260 | UpgradeLog*.htm 261 | ServiceFabricBackup/ 262 | *.rptproj.bak 263 | 264 | # SQL Server files 265 | *.mdf 266 | *.ldf 267 | *.ndf 268 | 269 | # Business Intelligence projects 270 | *.rdl.data 271 | *.bim.layout 272 | *.bim_*.settings 273 | *.rptproj.rsuser 274 | *- [Bb]ackup.rdl 275 | *- [Bb]ackup ([0-9]).rdl 276 | *- [Bb]ackup ([0-9][0-9]).rdl 277 | 278 | # Microsoft Fakes 279 | FakesAssemblies/ 280 | 281 | # GhostDoc plugin setting file 282 | *.GhostDoc.xml 283 | 284 | # Node.js Tools for Visual Studio 285 | .ntvs_analysis.dat 286 | node_modules/ 287 | 288 | # Visual Studio 6 build log 289 | *.plg 290 | 291 | # Visual Studio 6 workspace options file 292 | *.opt 293 | 294 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 295 | *.vbw 296 | 297 | # Visual Studio 6 auto-generated project file (contains which files were open etc.) 298 | *.vbp 299 | 300 | # Visual Studio 6 workspace and project file (working project files containing files to include in project) 301 | *.dsw 302 | *.dsp 303 | 304 | # Visual Studio 6 technical files 305 | *.ncb 306 | *.aps 307 | 308 | # Visual Studio LightSwitch build output 309 | **/*.HTMLClient/GeneratedArtifacts 310 | **/*.DesktopClient/GeneratedArtifacts 311 | **/*.DesktopClient/ModelManifest.xml 312 | **/*.Server/GeneratedArtifacts 313 | **/*.Server/ModelManifest.xml 314 | _Pvt_Extensions 315 | 316 | # Paket dependency manager 317 | .paket/paket.exe 318 | paket-files/ 319 | 320 | # FAKE - F# Make 321 | .fake/ 322 | 323 | # CodeRush personal settings 324 | .cr/personal 325 | 326 | # Python Tools for Visual Studio (PTVS) 327 | __pycache__/ 328 | *.pyc 329 | 330 | # Cake - Uncomment if you are using it 331 | # tools/** 332 | # !tools/packages.config 333 | 334 | # Tabs Studio 335 | *.tss 336 | 337 | # Telerik's JustMock configuration file 338 | *.jmconfig 339 | 340 | # BizTalk build output 341 | *.btp.cs 342 | *.btm.cs 343 | *.odx.cs 344 | *.xsd.cs 345 | 346 | # OpenCover UI analysis results 347 | OpenCover/ 348 | 349 | # Azure Stream Analytics local run output 350 | ASALocalRun/ 351 | 352 | # MSBuild Binary and Structured Log 353 | *.binlog 354 | 355 | # NVidia Nsight GPU debugger configuration file 356 | *.nvuser 357 | 358 | # MFractors (Xamarin productivity tool) working folder 359 | .mfractor/ 360 | 361 | # Local History for Visual Studio 362 | .localhistory/ 363 | 364 | # Visual Studio History (VSHistory) files 365 | .vshistory/ 366 | 367 | # BeatPulse healthcheck temp database 368 | healthchecksdb 369 | 370 | # Backup folder for Package Reference Convert tool in Visual Studio 2017 371 | MigrationBackup/ 372 | 373 | # Ionide (cross platform F# VS Code tools) working folder 374 | .ionide/ 375 | 376 | # Fody - auto-generated XML schema 377 | FodyWeavers.xsd 378 | 379 | # VS Code files for those working on multiple tools 380 | .vscode/* 381 | !.vscode/settings.json 382 | !.vscode/tasks.json 383 | !.vscode/launch.json 384 | !.vscode/extensions.json 385 | *.code-workspace 386 | 387 | # Local History for Visual Studio Code 388 | .history/ 389 | 390 | # Windows Installer files from build outputs 391 | *.cab 392 | *.msi 393 | *.msix 394 | *.msm 395 | *.msp 396 | 397 | # JetBrains Rider 398 | *.sln.iml 399 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [project-title] Changelog 2 | 3 | 4 | # x.y.z (yyyy-mm-dd) 5 | 6 | *Features* 7 | * ... 8 | 9 | *Bug Fixes* 10 | * ... 11 | 12 | *Breaking Changes* 13 | * ... 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to [project-title] 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 6 | 7 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | 11 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 12 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 13 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 14 | 15 | - [Code of Conduct](#coc) 16 | - [Issues and Bugs](#issue) 17 | - [Feature Requests](#feature) 18 | - [Submission Guidelines](#submit) 19 | 20 | ## Code of Conduct 21 | Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 22 | 23 | ## Found an Issue? 24 | If you find a bug in the source code or a mistake in the documentation, you can help us by 25 | [submitting an issue](#submit-issue) to the GitHub Repository. Even better, you can 26 | [submit a Pull Request](#submit-pr) with a fix. 27 | 28 | ## Want a Feature? 29 | You can *request* a new feature by [submitting an issue](#submit-issue) to the GitHub 30 | Repository. If you would like to *implement* a new feature, please submit an issue with 31 | a proposal for your work first, to be sure that we can use it. 32 | 33 | * **Small Features** can be crafted and directly [submitted as a Pull Request](#submit-pr). 34 | 35 | ## Submission Guidelines 36 | 37 | ### Submitting an Issue 38 | Before you submit an issue, search the archive, maybe your question was already answered. 39 | 40 | If your issue appears to be a bug, and hasn't been reported, open a new issue. 41 | Help us to maximize the effort we can spend fixing issues and adding new 42 | features, by not reporting duplicate issues. Providing the following information will increase the 43 | chances of your issue being dealt with quickly: 44 | 45 | * **Overview of the Issue** - if an error is being thrown a non-minified stack trace helps 46 | * **Version** - what version is affected (e.g. 0.1.2) 47 | * **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you 48 | * **Browsers and Operating System** - is this a problem with all browsers? 49 | * **Reproduce the Error** - provide a live example or a unambiguous set of steps 50 | * **Related Issues** - has a similar issue been reported before? 51 | * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be 52 | causing the problem (line of code or commit) 53 | 54 | You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/[organization-name]/[repository-name]/issues/new]. 55 | 56 | ### Submitting a Pull Request (PR) 57 | Before you submit your Pull Request (PR) consider the following guidelines: 58 | 59 | * Search the repository (https://github.com/[organization-name]/[repository-name]/pulls) for an open or closed PR 60 | that relates to your submission. You don't want to duplicate effort. 61 | 62 | * Make your changes in a new git fork: 63 | 64 | * Commit your changes using a descriptive commit message 65 | * Push your fork to GitHub: 66 | * In GitHub, create a pull request 67 | * If we suggest changes then: 68 | * Make the required updates. 69 | * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): 70 | 71 | ```shell 72 | git rebase master -i 73 | git push -f 74 | ``` 75 | 76 | That's it! Thank you for your contribution! 77 | -------------------------------------------------------------------------------- /Demos/.env: -------------------------------------------------------------------------------- 1 | GITHUB_TOKEN= 2 | AZURE_INFERENCE_SDK_KEY= -------------------------------------------------------------------------------- /Demos/Extensions/AI-Toolkit-Extension/Assets/MCP_command.txt: -------------------------------------------------------------------------------- 1 | npx -y @modelcontextprotocol/server-filesystem /Users//` -------------------------------------------------------------------------------- /Demos/Extensions/AI-Toolkit-Extension/Assets/brand-style-guide.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Demos/Extensions/AI-Toolkit-Extension/Assets/brand-style-guide.docx -------------------------------------------------------------------------------- /Demos/Extensions/AI-Toolkit-Extension/Assets/cta-evaluation-data.jsonl: -------------------------------------------------------------------------------- 1 | {"query":"Write a LinkedIn post encouraging developers to explore AI pair programming.", "response":"Still coding solo? AI pair programming with tools like GitHub Copilot can catch bugs, suggest functions, and boost your velocity. This week, try pairing GitHub Copilot with your current project and drop a takeaway in the comments—what worked, what didn’t?"} 2 | {"query":"Make a social post about how AI helps speed up prototyping.", "response":"Rapid prototyping is easier than ever with AI. Tools like GitHub Copilot let you turn ideas into code in minutes. Give it a try with your next side project."} 3 | {"query":"Draft a post about integrating AI into the dev workflow.", "response":"AI isn’t just for final drafts—it’s your dev assistant from planning to testing. Set a timer for 15 minutes, write a prompt for your current coding task, and see how GitHub Copilot helps. Share your prompt/output combo below to help other devs!"} -------------------------------------------------------------------------------- /Demos/Extensions/AI-Toolkit-Extension/Assets/cta-evaluator.md: -------------------------------------------------------------------------------- 1 | system: 2 | # Definition 3 | 4 | **Call to Action Quality** evaluates how clear, relevant, and motivating the call to action is in the model’s response, based on a 1–5 scale. 5 | 6 | Score 1: There is no call to action, or the one included is confusing, irrelevant, or misleading. 7 | Score 5: The CTA is crystal clear, action-oriented, well-aligned with the content, and highly motivating. It uses strong yet natural language and gives the reader a compelling reason to take action. 8 | 9 | The examples below show the Call to Action Quality score with reason for a question and response. 10 | 11 | **Example 1** 12 | question: How can I start learning about AI? 13 | response: That’s up to you. Everyone learns differently. 14 | output: {"score": 1, "reason": "The response avoids giving guidance and lacks any clear or motivating call to action."} 15 | 16 | **Example 2** 17 | question: How can I improve my resume for tech roles? 18 | response: Try using strong action verbs and highlight measurable impact. You can also book a free resume review session this week to get expert feedback. 19 | output: {"score": 5, "reason": "The CTA is clear, specific, and directly aligned with the user's goal—offering a next step that adds immediate value."} 20 | 21 | **Here the actual conversation to be scored:** 22 | question: {{query}} 23 | predicted answer: {{response}} 24 | output: -------------------------------------------------------------------------------- /Demos/Extensions/AI-Toolkit-Extension/Assets/single-post-schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "LinkedInSinglePostOutput", 3 | "strict": true, 4 | "schema": { 5 | "type": "object", 6 | "properties": { 7 | "text": { 8 | "type": "string" 9 | }, 10 | "hashtags": { 11 | "type": "array", 12 | "items": { 13 | "type": "string" 14 | } 15 | } 16 | }, 17 | "required": [ 18 | "text", 19 | "hashtags" 20 | ], 21 | "additionalProperties": false 22 | } 23 | } 24 | 25 | 26 | -------------------------------------------------------------------------------- /Demos/Extensions/AI-Toolkit-Extension/demo-aitk-create-agent-mcp-tools.md: -------------------------------------------------------------------------------- 1 | # Create an Agent with Tools from a MCP Server 2 | 3 | **Duration**: 10 mins 4 | 5 | This demo demonstrates how to create an AI agent with tools from an MCP (Model Context Protocol) server using the [AI Toolkit](https://aka.ms/AITookit) extension for Visual Studio Code. It includes step-by-step instructions for setting up the environment, adding the GPT-4o model, configuring an MCP server, and integrating tools into the agent. The demo also covers creating system prompts, running the agent, structuring model output, and saving results to a file system. 6 | 7 | ## Prerequisites 8 | 9 | - [Visual Studio Code](https://code.visualstudio.com/) 10 | - [AI Toolkit for Visual Studio Code](https://aka.ms/AIToolkit) 11 | - [GitHub Fine-grained personal access token (PAT)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) 12 | 13 | ## Setup Instructions 14 | 15 | ### Add the OpenAI GPT-4o model 16 | 17 | The demo leverages the **GPT-4o** model as the chosen model for the agent. The model should be added to **My Models** before running the demo. 18 | 19 | 1. Open the **AI Toolkit** extension from the **Activity Bar**. 20 | 1. In the **Catalog** section, select **Models** to open the **Model Catalog**. Selecting **Models** opens the **Model Catalog** in a new editor tab. 21 | 1. In the **Model Catalog** search bar, enter **OpenAI GPT-4o**. 22 | 1. Click **+ Add** to add the model to your **My Models** list. Ensure that you've selected the model that's **Hosted by GitHub**. 23 | 1. In the **Activity Bar**, confirm that the **OpenAI GPT-4o** model appears in the list. 24 | 25 | ### Prepare the MCP server command 26 | 27 | The demo leverages the [Filesystem](https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem) MCP server to write to a directory. Before running the demo, create a new `demo` directory on your desktop (or another easily accessible location) for the agent to save new files. Once the directory is created, take note of the **file path** as it'll be required for the MCP server command. 28 | 29 | For ease of preparation, the `MCP_command.txt` file contains the command that'll be required for adding the Filesystem MCP server. Open the file and replace the following placeholders: 30 | 31 | - `username` = Your system's user name (ex: `janedoe`) 32 | - `directory_location` = File path to the directory (ex: `Desktop/demo`) 33 | 34 | Provided below is an example of the command: 35 | 36 | `npx -y @modelcontextprotocol/server-filesystem /Users/janedoe/Desktop/demo` 37 | 38 | ## Running the Demo 39 | 40 | In this demo, you’ll show how to build, run, and enhance an AI agent inside Visual Studio Code using the AI Toolkit. This walkthrough covers prompt authoring, memory simulation, tool integration via MCP servers, and structured output—all within a single, cohesive workflow. 41 | 42 | ### Create an agent 43 | 44 | Start by introducing your audience to the **Agent (Prompt) Builder**—a space where they can create and customize their own AI-powered agents. In this section, you’ll show the audience how to create a new agent, give it a name, and assign a model like GPT-4o to power the conversation. 45 | 46 | 1. Open the **AI Toolkit** extension from the **Activity Bar**. 47 | 1. In the **Tools** section, select **Agent (Prompt) Builder**. Selecting **Agent (Prompt) Builder** opens the **Agent (Prompt) Builder** in a new editor tab. 48 | 1. Click the **+ New Builder** button. The extension will launch a setup wizard via the **Command Palette**. 49 | 1. Enter the name **Content Agent** and press **Enter**. 50 | 1. In the **Agent (Prompt) Builder**, for the **Model** field, select the **OpenAI GPT-4o (via GitHub)** model. 51 | 52 | ### Create a system prompt for the agent 53 | 54 | With the agent scaffolded, it’s time to define its personality and purpose. In this section, you’ll demonstrate how to use the **Generate system prompt** feature to describe the agent’s intended behavior—in this case, a content creator agent—and have the model write the system prompt for you. 55 | 56 | 1. For the **Prompts** section, click the **Generate system prompt** button. This button opens in the prompt builder which leverages AI to generate a system prompt for the agent. 57 | 1. In the **Generate a prompt** window, enter the following: `I want to create an agent that creates social media content across the following mediums: blogs, video scripts and captions for posts.` 58 | 1. Click the **Generate** button. A notification will appear in the bottom-right corner confirming that system prompt is being generated. Once the prompt generation is complete, the prompt will appear in the **System prompt** field of the **Agent (Prompt) Builder**. 59 | 1. Review the **System prompt** and modify if necessary. 60 | 61 | ### Run the agent 62 | 63 | Now that the system prompt is set, you’ll guide the audience through running the agent with a user prompt, reviewing the response, and then simulating conversation history by adding model responses back into the prompt sequence. This shows how the agent can “remember” past messages for a more contextual chat experience. 64 | 65 | 1. In the **User prompt** field, enter the following prompt: `Create a short LinkedIn post about developer productivity with AI tools`. 66 | 1. Click the **Run** button to generate the agent's response. 67 | 1. On the right side of the **Agent (Prompt) Builder**, review the **Model Response**. 68 | 1. Under the **Model Response**, click **Add to Prompts** to append the model's output to the prompts. This action add's the **Model Response** as an **Assistant prompt** and simulates maintaining conversational context with the agent. When you submit another **User Prompt** and run the agent again, the previously added **Model Response** will be included in the prompt, allowing the model to reference it as part of the conversation history. (*Note*: Adding the **Model Response** to prompts automatically adds a new blank **User prompt**.) 69 | 1. In the newly added **User prompt** field, enter the following prompt as a follow-up to the agent's response: `Revise to be more casual`. 70 | 1. Click the **Run** button to generate the agent's response. 71 | 1. Review the **Model Response**. 72 | 1. Under the **Model Response**, click **Add to Prompts** to append the model's output to the prompts. 73 | 74 | ### Add an MCP server 75 | 76 | Up until now, your agent has only generated text. In this section, you’ll introduce the audience to MCP servers, which allow the agent to use external tools. You’ll demonstrate how to connect to a local file-based server using a terminal command, bringing real-world actions into your agent's reach. 77 | 78 | Once the MCP server is connected, you'll show how to select tools from the server and integrate them into your agent. These tools allow the agent to take actions like writing files or performing lookups, enabling task completion beyond just text generation. 79 | 80 | 1. In the **Tools** section of the **Agent (Prompt) Builder**, click the **+ MCP Server** button. The extension will launch a setup wizard via the **Command Palette**. 81 | 1. Select **+ Add Server**. 82 | 1. Select **Connect to an Existing MCP Server**. 83 | 1. Select **Command (stdio)**. 84 | 1. Enter the following command to run: `npx -y @modelcontextprotocol/server-filesystem /Users//`. (Note: Replace `` with your system's user name. Replace `` with the file path to the directory.) 85 | 1. Enter the following name for the server: **Filesystem**. 86 | 1. Select the following tools and click **OK**: 87 | 1. read_file 88 | 1. write_file 89 | 1. edit_file 90 | 1. Review the tools added by opening the **Tools List** within the **Tools** section. (*Note*: To modify the server command, click the server name in the **Tools** section to access its `mcp.json` file.) 91 | 92 | ### Run the agent with tools 93 | 94 | Now that your agent has tools, it's time to use them. This section demonstrates how the agent can respond to a prompt by both generating content and taking action—like writing a file to your system. You’ll show the audience how to find and verify that file on their machine. 95 | 96 | 1. In the newly added **User prompt**, enter the following: `Create 5 related short LinkedIn posts about the benefits of using AI tools for developer productivity. Save the posts in a new file called "AI-productivity-series"`. (*Note*: If you did not add the prior **Model Response**, you can manually add a new **User prompt** field. Navigate to the top of the **Agent (Prompt) Builder**, click the **Add Prompt** button and select **User prompt**.) 97 | 1. Click the **Run** button to generate the agent's response. 98 | 1. Review the agent output. 99 | 1. In your file system, navigate to the directory that was chosen for the MCP Server command. (*Note*: If successful, the **Tool Response** provides the location in which the new file was written. The initial model response may insist that access to the path is denied, however, the follow-up model response should indicate that it was able to write to the directory.) 100 | 1. In the directory, validate that the agent created the file. 101 | 1. Open the file to review its content. 102 | 103 | ### Structure the model output 104 | 105 | Up to this point, your output has been unstructured text. This final section demonstrates how to format the model’s response using a JSON schema, ensuring structured outputs that can be easily parsed or consumed by other systems. This is especially useful for API responses, content formatting, and automated pipelines. 106 | 107 | 1. Open the `single-post-schema.json` file to view it's structure. Notice that the structure includes a section for the post and its corresponding hashtags. 108 | 1. In the **Structure output** section of the **Agent (Prompt) Builder**, click the **Choose output format** drop-down and select **json_schema**. 109 | 1. Click the **Prepare schema** button. The extension will launch a setup wizard via the **Command Palette**. 110 | 1. Select **Select local file**. 111 | 1. In the file explorer, select the `single-post-schema.json` file and click **Open**. 112 | 1. In the **Agent (Prompt) Builder**, delete the following user prompts: 113 | 1. `Revise to be more casual` 114 | 1. `Create 5 related short LinkedIn posts about the benefits of using AI tools for developer productivity. Save the posts in a new file called "AI-productivity-series.` 115 | 1. Navigate to the top of the **Agent (Prompt) Builder**, click the **Add Prompt** button and select **User prompt**. 116 | 1. In the newly added **User prompt**, enter the following: `Write a short LinkedIn post about AI developer productivity`. 117 | 1. Click the **Run** button to generate the agent's response. 118 | 1. Review the **Model Response** to validate the json schema format. 119 | 120 | ## Clean-up 121 | 1. In the **AI Toolkit** panel (Side Bar), in the **Tools** section, select **Agent (Prompt) Builder**. 122 | 1. At the top left of the **Agent (Prompt) Builder**, select the **Recent** button. 123 | 1. Select the **trash** icon next to the agent. 124 | 1. In the **Confirm Prompt Deletion** pop-up, select **Delete**. -------------------------------------------------------------------------------- /Demos/Extensions/AI-Toolkit-Extension/demo-aitk-evaluate-model-responses.md: -------------------------------------------------------------------------------- 1 | # Evaluate Model Responses 2 | 3 | **Duration**: 10 mins 4 | 5 | This demo focuses on evaluating AI model responses using the [AI Toolkit](https://aka.ms/AITookit) extension for Visual Studio Code. It demonstrates how to set up and run evaluations, create custom evaluators, and analyze results. The demo leverages the GPT-4o model as a judge for evaluations and provides step-by-step instructions for configuring evaluators, importing datasets, and reviewing evaluation outcomes. 6 | 7 | ## Prerequisites 8 | 9 | - [Visual Studio Code](https://code.visualstudio.com/) 10 | - [AI Toolkit for Visual Studio Code](https://aka.ms/AIToolkit) 11 | - [GitHub Fine-grained personal access token (PAT)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) 12 | 13 | ## Setup Instructions 14 | 15 | This section takes you through the steps for showcasing how the AI Toolkit helps evaluate model outputs using built-in and custom evaluation tools. You’ll guide the audience through setting up evaluations, creating custom evaluators, and analyzing results—all within Visual Studio Code. 16 | 17 | ### Add the OpenAI GPT-4o model 18 | 19 | The demo leverages the **GPT-4o** model as the model judge for evaluations. The model should be added to **My Models** before running the demo. 20 | 21 | 1. Open the **AI Toolkit** extension from the **Activity Bar**. 22 | 1. In the **Catalog** section, select **Models** to open the **Model Catalog**. Selecting **Models** opens the **Model Catalog** in a new editor tab. 23 | 1. In the **Model Catalog** search bar, enter **OpenAI GPT-4o**. 24 | 1. Click **+ Add** to add the model to your **My Models** list. Ensure that you've selected the model that's **Hosted by GitHub**. 25 | 1. In the **Activity Bar**, confirm that the **OpenAI GPT-4o** model appears in the list. 26 | 27 | ## Running the Demo 28 | 29 | ### Setup and run an evaluation 30 | 31 | In this section, you’ll take the audience through how to set up and run a standard evaluation using built-in evaluators. This helps demonstrate how developers can assess the quality of model outputs using tools like Likert scoring, pass/fail thresholds, or retrieval accuracy—all without leaving VS Code. 32 | 33 | 1. Open the **AI Toolkit** extension from the **Activity Bar**. 34 | 1. In the **Tools** section, select **Evaluation**. Selecting the **Evaluation** tool opens it in a new editor tab. 35 | 1. In the **Overview** tab of the **Evaluation** editor, click the **+ New Evaluation** button. The extension will launch a setup wizard via the **Command Palette**. 36 | 1. Enter the name **demo-evaluation** and press **Enter**. 37 | 1. Select the following evaluators and click **OK**: 38 | 1. Coherence 39 | 1. Fluency 40 | 1. Relevance 41 | 1. Similarity 42 | 1. Select the **gpt-4o GitHub** model. 43 | 1. Select **Sample dataset**. 44 | 1. A notification will appear in the bottom-right corner confirming that the evaluation was successfully created. Click the **Open demo-evaluation** button in the notification to view the evaluation setup. (*Note*: Alternatively, you can view the evaluation setup by navigating to the **Evaluation** editor and selecting the evaluation from the **Overview** tab.) 45 | 1. In the **Evaluation** editor, click the **Run Evaluation** button to start the evaluation job. When the evaluation run is complete, the results will appear in the editor. (*Note*: When the evaluation is run, a notification will appear in the bottom-right corner indicating the status of the job.) Once the job is complete, a **View Results** button will appear in the notification. Click the **View Results** button to view the evaluation results. (*Note*: Alternatively, you can view the evaluation results by navigating to the **Evaluation** editor and selecting the evaluation from the **Overview** tab.) 46 | 1. Review the evaluation results. 47 | 48 | ### Create a custom evaluator 49 | 50 | Now that the audience has seen how to run evaluations with built-in evaluators, this section introduces the power of custom evaluators. You’ll show how to create one from scratch by providing a name, description, evaluation type, and prompt—enabling tailored assessment criteria for specific scenarios. The custom evaluator will be used to assess the model's "Call to Action" for social posts. 51 | 52 | 1. In the **Evaluation** editor, select the **Evaluators** tab. (*Note*: Alternatively, if you're starting in the evaluation results view, you can click the **<** button in the Editor) 53 | 1. In the **Evaluators** tab, click the **+ Create Evaluator** button. 54 | 1. Enter the following for the custom evaluator: 55 | 1. Name - `Call to Action Quality` 56 | 1. Description - `Assess how clear, relevant, and motivating the call to action is in responses.` 57 | 1. Type - LLM-based 58 | 1. Prompt - Copy + paste the prompt within the `cta-evaluator.md` file. 59 | 1. Click **Save**. 60 | 61 | ### Setup and run an evaluation with a custom evaluator 62 | 63 | In this final section, you’ll demonstrate how to use the custom evaluator created in the previous step. This includes importing a dataset, mapping the relevant columns, and launching the evaluation—giving the audience a complete look at how to run bespoke model assessments inside the toolkit. 64 | 65 | 1. In the **Evaluation** editor, select the **Overview** tab and click the **+ New Evaluation** button. 66 | 1. In the setup wizard, enter the name **cta-evaluation** and press **Enter**. 67 | 1. Select the **Call to Action Quality** evaluator and click **OK**. 68 | 1. Select the **gpt-4o GitHub** model. 69 | 1. Select **Import dataset**. 70 | 1. In the file explorer, select the **cta-evaluation-data.jsonl** file. 71 | 1. A notification will appear in the bottom-right corner confirming that the evaluation was successfully created. Click the **Open cta-evaluation** button in the notification to view the evaluation setup. (*Note*: Alternatively, you can view the evaluation setup by navigating to the **Evaluation** editor and selecting the evaluation from the **Overview** tab.) 72 | 1. In the **Evaluation** editor, validate that the **query** and **response** column mappings are accurate. 73 | 1. Click the **Run Evaluation** button to start the evaluation job. 74 | 1. Review the evaluation results. 75 | 76 | ## Clean-up 77 | 1. In the **AI Toolkit** panel (Side Bar), in the **Tools** section, select **Evaluation**. 78 | 1. In the **Overview** tab of the **Evaluation** editor, check the boxes next to the **demo-evaluation** and **cta-evaluation** evaluation runs and click the **Delete** button. -------------------------------------------------------------------------------- /Demos/Extensions/AI-Toolkit-Extension/demo-aitk-explore-compare-models.md: -------------------------------------------------------------------------------- 1 | # Explore and Compare Models 2 | 3 | **Duration**: 5 mins 4 | 5 | This demo guides users through exploring, deploying, and comparing AI models using the [AI Toolkit](https://aka.ms/AITookit) extension for Visual Studio Code. It includes instructions for browsing the Model Catalog, chatting with models hosted by GitHub models, adding local models via Ollama, comparing responses from different models, and generating sample code to programmatically interact with models. 6 | 7 | ## Prerequisites 8 | 9 | - [Visual Studio Code](https://code.visualstudio.com/) 10 | - [AI Toolkit for Visual Studio Code](https://aka.ms/AIToolkit) 11 | - [GitHub Fine-grained personal access token (PAT)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) 12 | - [Ollama](https://ollama.com/) (with a minimum of 1 model pulled to your local Ollama library - i.e. Qwen, Gemma 3, Llama, etc.) 13 | 14 | ## Setup Instructions 15 | 16 | 1. Create and activate a virtual environment for your workspace. 17 | 1. Run `pip install -r requirements.txt` to install dependencies. 18 | 1. Add your [GitHub PAT](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) to the `.env` file. The PAT is required for making an inference call using the GPT-4o model hosted by GitHub Models in the **Generate and run sample code** section of the demo. 19 | 20 | ## Running the Demo 21 | 22 | This demo showcases how to use the AI Toolkit extension for Visual Studio Code to explore, test, and integrate language models—both cloud-hosted and local—into a content creation workflow. 23 | 24 | ### Browse models and view model cards 25 | 26 | In this section, you’ll introduce the audience to the Model Catalog within the AI Toolkit. You’ll show how to filter models by publisher and open a model card to learn more about a specific model’s capabilities, use cases, and configuration options. 27 | 28 | 1. Open the **AI Toolkit** extension from the **Activity Bar**. 29 | 1. In the **Catalog** section, select **Models** to open the **Model Catalog**. Selecting **Models** opens the **Model Catalog** in a new editor tab. Scroll through the **Model Catalog** to view the available models. 30 | 1. In the **Model Catalog**, use the **Publisher** filter and select **OpenAI**. 31 | 1. Select the **OpenAI GPT-4o** model name to view it's model card. The model card will open in a markdown file. View the contents of the model card. 32 | 33 | ### Add and chat with a model 34 | 35 | Now that your audience has seen how to find a model, it’s time to show them how to interact with one. In this section, you’ll demonstrate how to add a model to the Playground, add custom context instructions, upload a file as context, and start a conversation with the model. 36 | 37 | 1. Go back to the **Model Catalog** tab and select **Try in Playground** for the **OpenAI GPT-4o** model. The model will add to **My Models** and the **Playground** opens in a new editor tab. 38 | 1. In the **Playground**, go to the **Model Preferences** section and make sure **OpenAI GPT-4o (via GitHub)** is selected in the **Model** field. 39 | 1. In the **Model Preferences** section, locate the **Context instructions** field and enter the following: `You are a skilled content creator who specializes in crafting engaging, clear, and audience-appropriate written material across formats like blogs, newsletters, social media posts, and video scripts. You adapt your tone based on the platform and purpose—whether it’s professional, conversational, playful, or educational. Always aim to make the content actionable, concise, and aligned with the intended audience's interests. When asked, you may suggest headlines, outlines, or rewrites to improve flow and clarity.` 40 | 1. In the **chat window**, type the following into the chat input and press **Enter** : `I’m launching a newsletter for tech professionals interested in AI. Write a short introductory paragraph that sets the tone for a weekly email that’s equal parts informative, inspiring, and practical.` 41 | 1. Review the model's response. 42 | 1. In the chat input, select the **Search File** icon (paperclip) and upload the `brand-style-guide.docx` file. 43 | 1. In the **chat window**, type the following into the chat input and press **Enter** : 44 | ``` 45 | Does this paragraph follow the uploaded style guide? If not, what should I change? 46 | 47 | In today's competitive landscape, businesses must leverage AI or risk falling behind. 48 | Artificial intelligence offers myriad possibilities for optimizing workflows and scaling operations. 49 | Reach out today to explore how your organization can begin this transformative journey. 50 | ``` 51 | 1. Review the model's response. 52 | 53 | ### Add and chat with a local model 54 | 55 | This section is all about showing that you’re not limited to hosted models. Here, you’ll take the audience through adding a local model via Ollama, launching it in the Playground, and sending it a test prompt—highlighting flexibility and offline use cases. 56 | 57 | 1. In the **AI Toolkit** panel (Side Bar), go to the **My Models** section, then hover over **My Models** and click the **+** icon that appears. The extension will launch a setup wizard via the **Command Palette**. 58 | 1. Select **Add Ollama Model** followed by **Select models from Ollama library**. 59 | 1. Select your preferred model from the **Ollama models library**,then click **OK**. 60 | 1. A notification will appear in the bottom-right corner confirming that the model was successfully added. Click the **Model Playground** button in the notification to test the model in the **Model Playground**. (*Note*: Alternatively, you can navigate to the **Model Playground** by selecting **Playground** under the **Tools** section in the AI Toolkit side bar.) 61 | 1. In the **Playground** tab, go to the **Model Preferences** section and make sure that the Ollama model is selected in the **Model** field. The model will reflect **(Local via Ollama)** at the end of its name. (*Note*: If the prior context instructions are not retained, re-add the context instructions for this demo.) 62 | 1. In the **chat window**, type the following into the chat input and press **Enter**: `I’m launching a newsletter for tech professionals interested in AI. Write a short introductory paragraph that sets the tone for a weekly email that’s equal parts informative, inspiring, and practical.` (*Note*: Generation speed will vary as it's dependent on your hardware.) 63 | 1. Review the model's response. 64 | 65 | ### Compare model response for 2 models 66 | 67 | After showing how to chat with individual models, this section demonstrates the **Compare** feature. You’ll show the audience how to run the same prompt against two models side-by-side, making it easy to evaluate output differences in real time. 68 | 69 | 1. At the top of the playground, click **Compare** and select the **OpenAI GPT-4o via GitHub** model to compare model output. The **Playground** will display two chat windows. 70 | 1. You only need to type your prompt into one chat window—your input will automatically appear in both. In either **chat window**, enter the following into the input field and press **Enter**: `Write a brief intro for a weekly AI newsletter aimed at busy tech professionals. It should feel smart, helpful, and motivating.` 71 | 1. Review the output from the models. 72 | 73 | ### Generate and run sample code 74 | 75 | In this final section, you’ll show how the AI Toolkit can help developers go from prototype to production. You’ll demonstrate how to auto-generate a code snippet for calling the model, customize it with your own prompt, and run it—all from within VS Code. 76 | 77 | 1. In the **OpenAI GPT-4o (via GitHub)** chat window, click the **Select this model** button. 78 | 1. At the top right of the **Playground**, click ** View Code** to generate a code file to use the model programmatically. 79 | 1. The extension will launch a setup wizard via the **Command Palette**. Select **OpenAI SDK** for the client SDK. A new python file will open with the code. 80 | 1. Save the file (Ctrl+Shift+S) with the file name `aitk-inference-demo.py`. 81 | 1. In the `aitk-inference-demo.py` file, add the following after the import statements: 82 | ```python 83 | from dotenv import load_dotenv 84 | load_dotenv(override=True) 85 | ``` 86 | 1. In the `aitk-inference-demo.py` file, change the `text` string value to `Suggest a headline and opening line for a blog post about how AI is changing developer workflows.`. 87 | 1. Save (Ctrl+S) and run the file. 88 | 1. Remove the output from the model. 89 | 90 | ## Clean-up 91 | 1. In the **AI Toolkit** panel (Side Bar), go to the **My Models** section. 92 | 1. Right-click the **OpenAI GPT-4o** model and select **Delete**. 93 | 1. Right-click your loaded **Ollama** model and select **Delete**. 94 | 1. In the **Explorer** panel, delete the `aitk-inference-demo.py` file. -------------------------------------------------------------------------------- /Demos/Extensions/AI-Toolkit-Extension/requirements.txt: -------------------------------------------------------------------------------- 1 | python-dotenv 2 | openai -------------------------------------------------------------------------------- /Demos/Extensions/Azure-AI-Foundry-Extension/Assets/bing_grounding.yml: -------------------------------------------------------------------------------- 1 | tools: 2 | - type: bing_grounding 3 | options: 4 | tool_connections: 5 | - >- 6 | /subscriptions//resourceGroups//providers/Microsoft.MachineLearningServices/workspaces//connections/ -------------------------------------------------------------------------------- /Demos/Extensions/Azure-AI-Foundry-Extension/Assets/system-prompt.md: -------------------------------------------------------------------------------- 1 | Create an AI content-generation agent capable of producing engaging social media content across three mediums: blogs, video scripts, and captions for posts. 2 | 3 | # Instructions 4 | 5 | 1. **Understand the Medium**: Tailor the content based on the medium—informative and storytelling tone for blogs, visual-centric and conversational for video scripts, and concise and impactful for captions. 6 | 7 | 2. **Identify Audience and Purpose**: Clearly determine the target audience for the content and its purpose—inform, entertain, or drive engagement. 8 | 9 | 3. **Customize for Platform**: Adapt the tone, format, and style depending on the specific social media platform (e.g., Instagram, Twitter, LinkedIn, YouTube, etc.). 10 | 11 | 4. **Incorporate Trends**: Include trending phrases, hashtags, or visual styles relevant to the topic and platform. 12 | 13 | 5. **Review for Accuracy & Virality**: Ensure factual accuracy while optimizing the content for shareability. 14 | 15 | # Steps 16 | 17 | 1. **Gather Input Details**: 18 | - Topic of the blog, video, or caption. 19 | - Intended audience and tone of the content (e.g., professional, casual, humorous). 20 | - Social media platform(s) for customization. 21 | - Call-to-action or engagement goal (if applicable). 22 | 23 | 2. **Content Generation**: 24 | - For blogs: Write a structured piece with an engaging introduction, clear sections, and a strong conclusion. 25 | - For video scripts: Include scripting for scenes, dialogue/voiceover, and hooks for viewer retention. 26 | - For captions: Keep it concise, include hashtags, and create a compelling message/prompt for interaction. 27 | 28 | 3. **Optimize for Platform**: 29 | - Ensure blog length, keywords, and formatting match SEO best practices for sharing. 30 | - Video scripts should account for timing, pacing, and call-to-actions for on-screen execution. 31 | - Captions should be aligned with the character count limit and tone of the platform (e.g., witty for Instagram, professional for LinkedIn). 32 | 33 | 4. **Proof and Deliver**: 34 | - Check for grammatical and stylistic consistency. 35 | - Deliver content in the specified format preferred by the user. 36 | 37 | # Output Format 38 | 39 | - **Blogs**: 40 | Output a well-structured blog consisting of: 41 | * A catchy headline. 42 | * Introduction. 43 | * Key sections with subheadings. 44 | * Conclusion with key takeaways and/or a call-to-action. 45 | Format: Markdown with headers (`#`, `##`, etc.). 46 | 47 | - **Video Scripts**: 48 | Provide a clear, actionable script consisting of: 49 | * Scene descriptions. 50 | * Voiceover/dialogue. 51 | * Instructions for visuals or actions. 52 | Format: Text with clear labels (Scene 1, Dialogue, Visual, etc.). 53 | 54 | - **Captions**: 55 | Supply concise, engaging captions with relevant hashtags: 56 | Format: Plain text with hashtags on the next line. 57 | 58 | # Examples 59 | 60 | **Example 1** - Blog 61 | _Input_: Topic: "How to Start a Home Garden", Audience: Young professionals, Tone: Motivational 62 | _Output_: 63 | ``` 64 | # How to Start a Home Garden: A Step-by-Step Guide for Urban Dwellers 65 | 66 | Starting a home garden can transform your living space into an oasis of green while saving money and benefiting the environment. Here's how you can get started. 67 | 68 | ## Why Home Gardening Matters 69 | [Details about benefits with motivational tone] 70 | 71 | ## Step 1: Choose the Right Space 72 | [Details about evaluating light, space, and materials] 73 | 74 | ## Step 2: Select Your Plants 75 | [Guidance on choosing low-maintenance greens or herbs for starters] 76 | 77 | ... 78 | 79 | ## Key Takeaways and Call to Action 80 | Remember, every great garden starts with the first seed. Start your journey today and connect with us on Instagram by tagging #UrbanGardeners! 81 | ``` 82 | 83 | **Example 2** - Video Script 84 | _Input_: Topic: "5-Minute Morning Stretches", Platform: YouTube 85 | _Output_: 86 | ``` 87 | **Scene 1** 88 | - Visual: Bright bedroom, upbeat music plays in the background. 89 | - Voiceover: "Looking for a quick morning routine to jumpstart your day? We’ve got you covered with these 5-minute stretches!" 90 | 91 | **Scene 2** 92 | - Visual: Close-up of legs stretching on the carpet. 93 | - Dialogue: [Instructor voiceover]: "First, reach forward to touch your toes and hold for ten seconds." 94 | 95 | ... 96 | 97 | **Ending** 98 | - Visual: Smiling person sitting cross-legged. 99 | - Voiceover: "Feeling better already? Share this video and subscribe for more quick tips!" 100 | ``` 101 | 102 | **Example 3** - Caption 103 | _Input_: Topic: "Morning Coffee Routine", Platform: Instagram 104 | _Output_: 105 | "Start your day right ☀️ with our ultimate coffee guide! ☕ What's your go-to morning brew? Tell us below! ⬇️ 106 | #MorningRituals #CoffeeTime #BrewBetter" 107 | 108 | # Notes 109 | 110 | - Ensure all outputs align with the platform specifications. 111 | - Content should use a tone that matches the provided audience/type to maximize engagement. 112 | - Review edge cases like ambiguous topics or multi-purpose content; clarify if necessary before generating. -------------------------------------------------------------------------------- /Demos/Extensions/Azure-AI-Foundry-Extension/demo-aifx-create-agent-tools.md: -------------------------------------------------------------------------------- 1 | # Create an Agent with Tools 2 | 3 | **Duration**: 7 mins 4 | 5 | This demo explains how to create and deploy an AI agent with tools, such as Grounding with Bing Search, using the [Azure AI Foundry](https://marketplace.visualstudio.com/items?itemName=TeamsDevApp.vscode-ai-foundry) extension for Visual Studio Code. It provides instructions for creating and deploying an agent to Azure AI Foundry. The demo also includes steps for interacting with the agent in the Agent Playground and reviewing execution threads. 6 | 7 | ## Prerequisites 8 | 9 | - [Visual Studio Code](https://code.visualstudio.com/) 10 | - [Azure AI Foundry extension](https://marketplace.visualstudio.com/items?itemName=TeamsDevApp.vscode-ai-foundry) 11 | - [An existing Azure AI Foundry project](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-projects?tabs=ai-studio) (note: The extension interacts with Azure AI Foundry at the project level.) 12 | - [Grounding with Bing Search](https://portal.azure.com/#create/Microsoft.BingGroundingSearch) resource 13 | 14 | ## Setup Instructions 15 | 16 | ### Set your default project 17 | 18 | The Azure AI Foundry extension interacts with Azure AI Foundry at the project level. Before running the demo, you'll need to sign-in to Azure and select your default project. 19 | 20 | 1. In Visual Studio Code, select the Azure icon from the **Activity Bar**. 21 | 1. In the **Resources** section, select **Sign in to Azure...**. You'll be prompted to sign-in. 22 | 1. Once you're signed-in, navigate to the **Resources** section and select your **Azure Subscription** followed by your **Resource Group**. 23 | 1. Select **Azure AI Foundry** 24 | 1. Right-click your project and select **Open in Azure AI Foundry Extension**. 25 | 26 | ### Connect the Grounding with Bing Search resource to your project 27 | 28 | The **Grounding with Bing Search** resource must be connected to the Azure AI Foundry project that'll be used for this demo. The connection can be made within the **Management Center** of your project. 29 | 30 | 1. Navigate to the [Azure AI Foundry portal](https://ai.azure.com) and select your project. 31 | 1. In the left navigation menu, towards the bottom, select **Management center**. 32 | 1. In the **Connected resources** section, select **New connection**. 33 | 1. In the **Add a connection to external assets** window, select **Grounding with Bing Search**. 34 | 1. In the **Connect a Grounding with Bing Search Account** window, select **Add connection** next to your **Grounding with Bing** resource. 35 | 1. Click **Close**. 36 | 37 | ### Prepare the Bing Grounding tool YAML 38 | 39 | For ease of preparation, the `bing_grounding.yml` file contains the yaml that'll later be required for adding **Grounding with Bing Search** as a tool for the agent. The YAML contains placeholder values that should be replaced prior to running the demo. The values for the placeholders are available within the **Overview** section of the **Management center** of your project. Open the file and replace the following placeholders: 40 | 41 | - `subscription_ID` = Your Subscription ID 42 | - `resource_group_name` = Your Resource Group name 43 | - `project_name` = Your Project name 44 | - `bing_grounding_connection_name` = The connection name **NOT** the resource name 45 | 46 | Provided below is an example of the `tool-connection` for `bing_grounding`: 47 | 48 | `/subscriptions/a4f7c123-9be2-4c88-a9df-8e3a6b75fc10/resourceGroups/rg-aifx-demo/providers/Microsoft.MachineLearningServices/workspaces/aifx-demo/connections/aifxbinggrounding` 49 | 50 | ## Running the Demo 51 | 52 | In this demo, you’ll show how to create, deploy, and interact with an AI agent using the Azure AI Foundry extension. You’ll take the audience through the full process—from configuring agent metadata to running it in the Playground and inspecting execution threads. 53 | 54 | ### Create an agent 55 | 56 | In this section, you’ll show how to create a new agent using the agent designer and YAML editor. You’ll configure basic metadata, assign a model, add instructions, and manually add a tool like Grounding with Bing Search. This highlights how both visual and code-based editing can be used together for agent configuration. 57 | 58 | 1. Open the **Azure AI Foundry** extension from the **Activity Bar**. 59 | 1. In the **Resources** section, within the **Agents** subsection, hover over **Agents** and click the **+** icon that appears. You will be prompted the save the agent file. 60 | 1. In the file save dialog that appears, navigate to the directory in which you want to save the agent file. Name the file `content-agent` and click **Save Agent File**. A notification will appear in the bottom-right corner confirming that the agent file was successfully saved. The agent YAML file and designer will open. 61 | 1. In the designer editor (right), enter the following: 62 | 1. Name: **Content Agent** 63 | 1. Model: **gpt-4o** 64 | 1. Instructions: Copy and paste the instructions from the `system-prompt.md` file. 65 | 1. In the agent YAML file (left), complete the following: 66 | 1. In the `metadata.authors` sequence, replace the `author1` value with your name and remove the `author2` value. 67 | 1. In the `metadata.tags` sequence, remove the values `tag1` and `tag2`. 68 | 1. Replace the `tools` key, with the entry in the **bing_grounding.yml** file (do not add via the designer). 69 | 1. Save the agent yaml file. 70 | 1. In the designer editor (right), validate that the **Bing Grounding** tool appears in the **Tool** section. 71 | 1. At the bottom of the designer editor, click the **Deploy to Azure AI Foundry** button. A notification will appear in the bottom-right corner confirming that the agent was successfully created. 72 | 73 | ### Run the agent 74 | 75 | After deploying the agent, this section takes the audience through running the agent in the Agent Playground. You’ll demonstrate how to submit a prompt, view the response, and optionally carry on the conversation—bringing your agent to life in a natural, chat-like interface. 76 | 77 | 1. In the left **Activity Bar**, navigate to the **Resources** section. 78 | 1. Expand the **Agents** subsection and confirm that the **Content Agent** appears in the list of agents. 79 | 1. In the left **Activity Bar**, navigate to the **Tools** section and select **Agent Playground**. The extension will launch a setup wizard via the **Command Palette**. 80 | 1. Select the agent. The **Agent Playground** will then open in a new editor tab. 81 | 1. In the **chat window**, type the following into the chat input and press **Enter** : `Find three recent blog posts or articles (published in the past month) about how AI is being used by developers. Summarize the key takeaways from each, and suggest one new content idea inspired by the trends you found.`. 82 | 1. Review the agent's response and submit a suitable follow-up prompt. 83 | 84 | ### View threads 85 | 86 | Finally, you'll show the audience how to inspect the execution history of the agent by reviewing its threads. This section focuses on exploring a specific response, expanding the step details, and gaining insight into how the agent processed the request—perfect for debugging and transparency. 87 | 88 | 1. In the left **Activity Bar**, navigate to the **Resources** section. 89 | 1. Expand the **Threads** subsection. 90 | 1. Select the first successful thread (indicated with a green circle and check-mark). 91 | 1. In the agent's response, expand the **Step details** and review the information provided. 92 | 93 | ## Clean-up 94 | 1. In the **Azure AI Foundry** panel (Side Bar), go to the **Resources** section. 95 | 1. Within the **Agents** subsection, right-click the **Content Agent** and select **Delete**. -------------------------------------------------------------------------------- /Demos/Extensions/Azure-AI-Foundry-Extension/demo-aifx-explore-compare-models.md: -------------------------------------------------------------------------------- 1 | # Explore and Compare Models 2 | 3 | **Duration**: 5 mins 4 | 5 | This demo showcases how to explore, deploy, and interact with AI models using the [Azure AI Foundry](https://marketplace.visualstudio.com/items?itemName=TeamsDevApp.vscode-ai-foundry) extension and the [AI Toolkit](https://aka.ms/AITookit) extension for Visual Studio Code. It provides step-by-step instructions for setting up the environment, browsing the model catalog, deploying models to Azure, and using the Playground to chat with models. The demo also covers comparing model responses, generating sample code for programmatic interaction, and managing model deployments, offering a comprehensive guide for leveraging Azure AI Foundry in AI development workflows. 6 | 7 | ## Prerequisites 8 | 9 | - [Visual Studio Code](https://code.visualstudio.com/) 10 | - [Azure AI Foundry extension](https://marketplace.visualstudio.com/items?itemName=TeamsDevApp.vscode-ai-foundry) 11 | - [AI Toolkit for Visual Studio Code](https://aka.ms/AIToolkit) 12 | - [An existing Azure AI Foundry project](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-projects?tabs=ai-studio) with a `o4-mini` deployment (note: The extension interacts with Azure AI Foundry at the project level.) 13 | 14 | ## Setup Instructions 15 | 16 | ### Setup environment and install dependencies 17 | 18 | 1. Create a virtual environment for your workspace. 19 | 1. Run `pip install -r requirements.txt` to install dependencies. 20 | 1. Add your **AZURE_INFERENCE_SDK_KEY** to the `.env` file. This key is available on the **Overview** page of your Azure AI Foundry project. 21 | 22 | ### Set your default project 23 | 24 | The Azure AI Foundry extension interacts with Azure AI Foundry at the project level. Before running the demo, you'll need to sign-in to Azure and select your default project. 25 | 26 | 1. In Visual Studio Code, select the Azure icon from the **Activity Bar**. 27 | 1. In the **Resources** section, select **Sign in to Azure...**. You'll be prompted to sign-in. 28 | 1. Once you're signed-in, navigate to the **Resources** section and select your **Azure Subscription** followed by your **Resource Group**. 29 | 1. Select **Azure AI Foundry** 30 | 1. Right-click your project and select **Open in Azure AI Foundry Extension**. 31 | 32 | ## Running the Demo 33 | 34 | In this demo, you’ll showcase how to discover, deploy, and interact with language models using the Azure AI Foundry extension. You'll cover the full lifecycle—from model browsing to code generation—using the built-in tools within Visual Studio Code. 35 | 36 | ### Browse models in the Model Catalog 37 | 38 | Begin by introducing your audience to the Model Catalog, a centralized place for discovering models across different publishers and tasks. This section shows how to browse available models and apply filters to find the right one for your needs. 39 | 40 | 1. Open the **Azure AI Foundry** extension from the **Activity Bar**. 41 | 1. In the **Tools** section, select **Model Catalog** to open the **Model Catalog** in a new editor tab. Scroll through the **Model Catalog** to view the available models. 42 | 1. Apply the filters to filter the models by either **Publisher** or **Task**. 43 | 44 | ### Deploy and chat with a model 45 | 46 | Now that your audience has seen where to find models, you’ll demonstrate how to deploy a model into your Azure project using the built-in wizard. After deployment, you’ll switch to the Model Playground to run a chat session, showing how developers can test responses interactively with custom prompts and file uploads. 47 | 48 | 1. In the **Model Catalog**, search for **gpt-4o** and select the **Deploy in Azure** link for the **gpt-4o** model. The extension will launch a setup wizard via the **Command Palette**. 49 | 1. Select your **AI service**. 50 | 1. Select your preferred **model version**. 51 | 1. Select your preferred **deployment type**. 52 | 1. Enter the following **name** for the model and press **Enter**: `gpt-4o` 53 | 1. A confirmation dialog box will appear. Click the **Deploy** button to deploy the model to your project. 54 | 1. In the left **Activity Bar**, navigate to the **Tools** section, and click **Model Playground**. The extension will launch a setup wizard via the **Command Palette**. 55 | 1. Select the **gpt-4o** model. The **Playground** will then open in a new editor tab. 56 | 1. In the **Playground**, go to the **Model Preferences** section and make sure **gpt-4o_AIFoundry (via Custom)** is selected in the **Model** field. 57 | 1. In the **Model Preferences** section, locate the **Context instructions** field and enter the following: `You are a skilled content creator who specializes in crafting engaging, clear, and audience-appropriate written material across formats like blogs, newsletters, social media posts, and video scripts. You adapt your tone based on the platform and purpose—whether it’s professional, conversational, playful, or educational. Always aim to make the content actionable, concise, and aligned with the intended audience's interests. When asked, you may suggest headlines, outlines, or rewrites to improve flow and clarity.` 58 | 1. Review the model's response. 59 | 1. In the **chat window**, type the following into the chat input and press **Enter** : `I’m launching a newsletter for tech professionals interested in AI. Write a short introductory paragraph that sets the tone for a weekly email that’s equal parts informative, inspiring, and practical.` 60 | 1. Review the model's response. 61 | 1. In the chat input, select the **Search File** icon (paperclip) and upload the `brand-style-guide.docx` file. 62 | 1. In the **chat window**, type the following into the chat input and press **Enter** : 63 | ``` 64 | Does this paragraph follow the uploaded style guide? If not, what should I change? 65 | 66 | In today's competitive landscape, businesses must leverage AI or risk falling behind. 67 | Artificial intelligence offers myriad possibilities for optimizing workflows and scaling operations. 68 | Reach out today to explore how your organization can begin this transformative journey. 69 | ``` 70 | 1. Review the model's response. 71 | 72 | ## Chat with an existing model deployment 73 | 74 | To show flexibility with existing resources, this section demonstrates how to view model deployment information for an existing deployed model directly from the Resources panel. You’ll take the audience through interacting with that model in the Playground, just like with a fresh deployment—without repeating the setup process. 75 | 76 | 1. In the left **Activity Bar**, navigate to the **Resources** section. 77 | 1. Within the **Models** subsection, right-click the **o4-mini** model and select **Open in Playground**. The **Playground** will then open in a new editor tab. 78 | 1. In the **Playground**, go to the **Model Preferences** section and make sure **o4-mini_AIFoundry (via Custom)** is selected in the **Model** field. 79 | 1. In the **Model Preferences** section, locate the **Context instructions** field and enter the following: `You are a skilled content creator who specializes in crafting engaging, clear, and audience-appropriate written material across formats like blogs, newsletters, social media posts, and video scripts. You adapt your tone based on the platform and purpose—whether it’s professional, conversational, playful, or educational. Always aim to make the content actionable, concise, and aligned with the intended audience's interests. When asked, you may suggest headlines, outlines, or rewrites to improve flow and clarity.` 80 | 1. In the **chat window**, type the following into the chat input and press **Enter** : `I’m launching a newsletter for tech professionals interested in AI. Write a short introductory paragraph that sets the tone for a weekly email that’s equal parts informative, inspiring, and practical.` 81 | 1. Review the model's response. 82 | 83 | ### Compare model response for 2 models 84 | 85 | In this section, you’ll highlight how multiple models can be compared side-by-side in the Playground. This is useful when teams are evaluating model quality, fine-tuning decisions, or vendor/model selection. You’ll enter a single prompt and show how the outputs differ in real time. 86 | 87 | 1. At the top of the playground, click **Compare** and select the **gpt-4o_AIFoundry (via Custom)** model to compare model output. The **Playground** will display two chat windows. 88 | 1. You only need to type your prompt into one chat window—your input will automatically appear in both. In either **chat window**, enter the following into the input field and press **Enter**: `Write a brief intro for a weekly AI newsletter aimed at busy tech professionals. It should feel smart, helpful, and motivating.` 89 | 1. Review the output from the models. 90 | 91 | ### Generate and run sample code 92 | 93 | Finally, you’ll show the audience how to generate and run inference code programmatically. This section shows how to export model usage as a Python script using the Azure AI client SDK—bridging the gap between experimentation and integration in an app or workflow. 94 | 95 | 1. In the left **Activity Bar**, navigate to the **Resources** section 96 | 1. Within the **Models** subsection, right-click the **gpt-4o** model and select **Open Code File**. The extension will launch a setup wizard via the **Command Palette**. 97 | 1. Select **Azure AI model inference client library** for the **SDK**. 98 | 1. Select **Python** for **language**. 99 | 1. Select **Key Authentication** for the **auth method**. 100 | 1. A new python file will open with the code. 101 | 1. Save the file (Ctrl+Shift+S) with the file name `aifx-inference-demo.py`. 102 | 1. In the `aifx-inference-demo.py` file, add the following after the import statements: 103 | ``` 104 | from dotenv import load_dotenv 105 | load_dotenv() 106 | ``` 107 | 1. In the `aifx-inference-demo.py` file, change the value for the `key` variable to `os.getenv("AZURE_INFERENCE_SDK_KEY")`. 108 | 1. Save (Ctrl+S) and run the file. (*Note*: You can modify the `print` statement to only output the model's response. To do so, replace `print(response)` with `print(response.choices[0].message.content)`) 109 | 1. Review the output from the model. 110 | 111 | ## Clean-up 112 | 1. In the **Azure AI Foundry** panel (Side Bar), go to the **Resources** section. 113 | 1. Within the **Models** subsection, right-click the **gpt-4o** model and select **Delete**. 114 | 1. In the **Explorer** panel, delete the **afx-inference-demo.py** file. -------------------------------------------------------------------------------- /Demos/Extensions/Azure-AI-Foundry-Extension/requirements.txt: -------------------------------------------------------------------------------- 1 | python-dotenv 2 | azure.identity 3 | azure-ai-inference -------------------------------------------------------------------------------- /Demos/README.md: -------------------------------------------------------------------------------- 1 | # AI Toolkit & Azure AI Foundry Extension Demos 2 | 3 | The following resources are intended for a presenter to learn and deliver AI Toolkit and Azure AI Foundry extension demos. 4 | 5 | ## AI Engineer Scenarios 6 | 7 | |Scenario |Description |Duration |Video | 8 | |---------|---------|---------|---------| 9 | |[Explore and Compare Models](Extensions/AI-Toolkit-Extension/demo-aitk-explore-compare-models.md) (AITK) | This demo guides users through exploring, deploying, and comparing AI models using the AI Toolkit extension for Visual Studio Code. It includes instructions for browsing the Model Catalog, chatting with models hosted by GitHub models, adding local models via Ollama, comparing responses from different models, and generating sample code to programmatically interact with models. | 5 mins | [Watch](https://microsoft.sharepoint.com/:v:/t/PyConUS/EaGl-GF_0z1BgZizovIvEoEBYwqT1JVXuWvAxBqlhdjpVQ?e=9piMaE) | 10 | |[Evaluate Model Responses](Extensions/AI-Toolkit-Extension/demo-aitk-evaluate-model-responses.md) (AITK) | This demo focuses on evaluating AI model responses using the AI Toolkit extension for Visual Studio Code. It demonstrates how to set up and run evaluations, create custom evaluators, and analyze results. The demo leverages the GPT-4o model as a judge for evaluations and provides step-by-step instructions for configuring evaluators, importing datasets, and reviewing evaluation outcomes. | 10 mins | [Watch](https://microsoft.sharepoint.com/:v:/t/PyConUS/EbjZcLEyupFBqv7royVyN-sBEAk6QeKaFYhURKvgbsDwfw?e=J0PRtu) | 11 | |[Create an Agent with Tools from a MCP Server](Extensions/AI-Toolkit-Extension/demo-aitk-create-agent-mcp-tools.md) (AITK) | This demo demonstrates how to create an AI agent with tools from an MCP (Model Context Protocol) server using the AI Toolkit extension for Visual Studio Code. It includes step-by-step instructions for setting up the environment, adding the GPT-4o model, configuring an MCP server, and integrating tools into the agent. The demo also covers creating system prompts, running the agent, structuring model output, and saving results to a file system. | 10 mins | [Watch](https://microsoft.sharepoint.com/:v:/t/PyConUS/EVJTzBn-dJZBh-rKtJd6j2UBzrhSsly8VbRlJVbLDMxeNA?e=zeKnYs) | 12 | |[Explore and Compare Models](Extensions/Azure-AI-Foundry-Extension/demo-aifx-explore-compare-models.md) (AIFX) | This demo showcases how to explore, deploy, and interact with AI models using the Azure AI Foundry extension and the AI Toolkit extension for Visual Studio Code. It provides step-by-step instructions for setting up the environment, browsing the model catalog, deploying models to Azure, and using the Playground to chat with models. The demo also covers comparing model responses, generating sample code for programmatic interaction, and managing model deployments, offering a comprehensive guide for leveraging Azure AI Foundry in AI development workflows. | 5 mins | [Watch](https://microsoft.sharepoint.com/:v:/t/PyConUS/EYWyzTHN2-lDuCJ0UZpcg0IB38IEMGYEDIy5kr_uvSjjnA?e=aKLuvl) | 13 | |[Create an Agent with Tools](Extensions/Azure-AI-Foundry-Extension/demo-aifx-create-agent-tools.md) (AIFX) | This demo explains how to create and deploy an AI agent with tools, such as Grounding with Bing Search, using the Azure AI Foundry extension for Visual Studio Code. It provides instructions for creating and deploying an agent to Azure AI Foundry. The demo also includes steps for interacting with the agent in the Agent Playground and reviewing execution threads. | 7 mins | [Watch](https://microsoft.sharepoint.com/:v:/t/PyConUS/Ecc6Opddt89Ihp6XIxVLSp4BL_EIlz_FbI_wHc1ZHoga2g?e=w3IUhm) | 14 | 15 | ## FAQs 16 | 17 | #### Where can I download the AI Toolkit? 18 | You can download and install the AI Toolkit within the **Extensions View** in Visual Studio Code. Alternatively, you can visit [aka.ms/AIToolkit](https://aka.ms/AIToolkit) to view the extension in the Visual Studio Marketplace. 19 | 20 | #### Where can I download the Azure AI Foundry extension? 21 | You can download and install the Azure AI Foundry extension within the **Extensions View** in Visual Studio Code. Alternatively, you can visit the [Visual Studio Marketplace](https://marketplace.visualstudio.com/items?itemName=TeamsDevApp.vscode-ai-foundry). 22 | 23 | #### Where can I find the documentation for the AI Toolkit? 24 | Documentation for the AI Toolkit is available at [aka.ms/AITookit/doc](https://aka.ms/AIToolkit/doc). 25 | 26 | #### Where can I find the documentation for the Azure AI Foundry extension? 27 | Documentation for the Azuer AI Foundry extension is available on Microsoft Learn: 28 | - [Work with the Azure AI Foundry for Visual Studio Code extension](https://aka.ms/aif-vscode-doc) 29 | - [Work with Azure AI Agent Service in Visual Studio Code](https://aka.ms/aif-vscode-agent-doc) 30 | 31 | #### Can I use my own models or other models from Hugging Face? 32 | If your own model supports the OpenAI API contract, you can host it in the cloud and [add the model to AI Toolkit](https://code.visualstudio.com/docs/intelligentapps/models) as a custom model. You need to provide key information such as model endpoint URL, access key and model name. 33 | 34 | #### Does the extension work in Linux or other systems? 35 | Yes, AI Toolkit runs on Windows, Mac, and Linux. 36 | 37 | #### Why does the AI Toolkit need GitHub and Hugging Face credentials? 38 | We host all the project templates in GitHub, and the base models are hosted in Azure or Hugging Face. These environments require an account for access via the APIs. 39 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # AI Toolkit Samples 3 | 4 | [AI Toolkit for Visual Studio Code](aka.ms/AIToolkit) is a powerful extension for Visual Studio Code that streamlines agent development. AI Engineers can easily build AI applications by developing and testing generative AI models—locally or in the cloud. The extension supports most major generative models available today. 5 | 6 | With AI Toolkit, AI Engineers can: 7 | 8 | - Explore and evaluate models from a wide range of providers—including Anthropic, OpenAI, GitHub—or run models locally using ONNX and Ollama 9 | 10 | - Build and test agents in minutes with prompt generation, quick starters, and seamless MCP tool integrations. 11 | 12 | - Fine-tune models on a local machine with GPU or in the cloud (Azure Container App) with GPU 13 | 14 | ## Try Our Samples 15 | The AI Toolkit [samples](Samples) are designed to help developers and researchers explore and implement AI solutions effectively. Our samples include: 16 | - Sample Code: Pre-built examples to demonstrate AI functionalities, such as training, deploying, or integrating models into applications. 17 | - Documentation: Guides and tutorials to help users understand AI Toolkit features and how to use them. 18 | 19 | **Prequisites** 20 | - [Visual Studio Code](https://code.visualstudio.com/) 21 | - [AI Toolkit for Visual Studio Code](https://aka.ms/AIToolkit) 22 | - [GitHub Fine-grained personal access token (PAT)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token) 23 | 24 | **Get Started** 25 | 1. Star and fork this repository. 26 | 1. In Visual Studio Code, clone your forked repository. You can do so via the Command Palette (CTRL/CMD+Shift+P) by first searching for **Git: Clone** and selecting your fork to clone locally. (*Note*: Alternatively, you can run the command `git clone https://github.com/Azure-Samples/AI_Toolkit_Samples.git` in the terminal.) 27 | 1. After cloning the repo, open the project folder in VS Code and navigate to the [Samples](samples) folder. 28 | 29 | ## AI Toolkit Blogs 30 | 31 | Want to dive deeper into what you can do with the AI Toolkit? Our blog articles cover everything from getting started to advanced use cases, including tips for working with local and cloud-based models, prompt engineering techniques, and evaluation strategies. 32 | 33 | Whether you're just exploring or ready to build production-ready AI apps, these posts are designed to help you get the most out of the extension. 34 | 35 | Check out the latest articles below to expand your skills and stay up to date with new features! 36 | 37 | | Article | Abstract | 38 | |------------------|--------------------| 39 | | [Visual Studio AI Toolkit : Building Phi-3 GenAI Applications](https://techcommunity.microsoft.com/blog/educatordeveloperblog/visual-studio-ai-toolkit--building-phi-3-genai-applications/4179355)| Learn how to run pre-optimized AI models locally, test and integrate them seamlessly, and fine-tune models for specific needs. | 40 | | [Expanded model catalog for AI Toolkit](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/expanded-model-catalog-for-ai-toolkit/4295917) | The AI Toolkit now supports a broader range of models, including those from Google, Anthropic, and GitHub, enhancing its capabilities for integrating AI into applications. | 41 | | [Bring your own models](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/bring-your-own-models-on-ai-toolkit---using-ollama-and-api-keys/4369411) | The AI Toolkit now supports local models via Ollama and remote models using API keys for OpenAI, Google, and Anthropic, expanding its versatility for developers. | 42 | |[Prompt Engineering Simplified: AI Toolkit's Prompt Builder](https://techcommunity.microsoft.com/blog/educatordeveloperblog/prompt-engineering-simplified-ai-toolkits-prompt-builder/4384783)| The Prompt Builder in AI Toolkit simplifies the creation and refinement of prompts for large language models, enabling users to generate, test, and customize prompts efficiently. | 43 | | [Building Retrieval Augmented Generation on VSCode & AI Toolkit](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/building-retrieval-augmented-generation-on-vscode--ai-toolkit/4241035) | Learn how to build a Retrieval-Augmented Generation (RAG) application using VS Code and AI Toolkit. | 44 | | [Building RAG on Phi-3 locally using embeddings on VS Code AI Toolkit](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/building-rag-on-phi-3-locally-using-embeddings-on-vs-code-ai-toolkit/4246839) |Learn how to build a Retrieval-Augmented Generation (RAG) application locally using the Phi-3 model and embeddings with the AI Toolkit. | 45 | | [How Reasoning Models are transforming Logical AI thinking](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/how-reasoning-models-are-transforming-logical-ai-thinking/4373194) | Explore how reasoning models excel at logical problem-solving by breaking down complex tasks into smaller steps and solving them through explicit logical reasoning. | 46 | | [Recipe Generator Application with Phi-3 Vision on AI Toolkit Locally](https://techcommunity.microsoft.com/blog/educatordeveloperblog/recipe-generator-application-with-phi-3-vision-on-ai-toolkit-locally/4366157) | Learn how to create a Recipe Generator Application using the Phi-3 Vision model and the AI Toolkit. | 47 | 48 | ## AI Sparks Series 49 | SLMs & Local Models – Test and deploy AI models and applications efficiently on your own terms locally, to edge devices or to the cloud 50 | Embedding Models & RAG – Supercharge retrieval for smarter applications using existing data. 51 | Multi-Modal AI – Work with images, text, and beyond. 52 | Agentic Frameworks – Build autonomous, decision-making AI systems. 53 | 54 | What will you learn from this session? 55 | Whether you're a developer, startup founder, or AI enthusiast, you'll gain practical insights, live demos, and actionable takeaways to level up your AI integration journey. 56 | 57 | Join us and spark your AI transformation - view the [AI Sparks](https://www.youtube.com/playlist?list=PLmsFUfdnGr3yysvu8fPA9ka5gW2fkJci1) playlist! 58 | 59 | ## Demo Resources for Presenters 60 | Thinking about including the AI Toolkit in a presentation? We’ve got you covered with a set of ready-to-go [demos](Demos) to help you learn and showcase both the AI Toolkit and Azure AI Foundry extensions. 61 | 62 | ## Join the Azure AI Foundry Discord 63 | [![Azure AI Foundry Discord](https://dcbadge.vercel.app/api/server/ByRwuEEgH4)](https://discord.com/invite/ByRwuEEgH4) 64 | 65 | ## AI Toolkit Updates 66 | | Website | Page Title | 67 | |------------------|--------------------| 68 | | [March Update](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/ai-toolkit-for-vs-code-march-update/4396880) | AI Toolkit March 2025 Update 69 | | [Feb Update](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/ai-toolkit-for-vs-code-february-update/4383280) | AI Tookit Feb 2025 Update | 70 | | [Jan Update](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/ai-toolkit-for-vs-code-january-update/4371592) | AI Toolkit Jan 2025 Update | 71 | | [Oct Update](https://techcommunity.microsoft.com/blog/azuredevcommunityblog/ai-toolkit-for-visual-studio-code-october-2024-update-highlights/4298718) | AI Toolkit Oct 2024 Update | 72 | 73 | ## Resources 74 | 75 | * [AI Toolkit for VSCode Documentation](https://learn.microsoft.com/en-us/windows/ai/toolkit/?WT.mc_id=aiml-149848-viheg) 76 | * [AI Toolkit for Visual Studio Code (Github)](https://github.com/microsoft/vscode-ai-toolkit/?WT.mc_id=aiml-149848-viheg) 77 | * [RAG in AI studio (Concepts)](https://learn.microsoft.com/en-us/azure/ai-studio/concepts/retrieval-augmented-generation/?WT.mc_id=aiml-149848-viheg) 78 | * [RAG in AI Search](https://learn.microsoft.com/en-us/azure/search/retrieval-augmented-generation-overview/?WT.mc_id=aiml-149848-viheg) 79 | * [Phi-3 Cookbook](https://github.com/microsoft/Phi-3CookBook/?WT.mc_id=aiml-149848-viheg) 80 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /Samples/1. Basic/AI_Toolkit_Streamlit.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | from openai import OpenAI 3 | 4 | client = OpenAI( 5 | base_url="http://127.0.0.1:5272/v1/", 6 | api_key="xyz" # required by API but not used 7 | ) 8 | 9 | st.title("Chat with Phi-3") 10 | query = st.chat_input("Enter query:") 11 | 12 | if query: 13 | with st.chat_message("user"): 14 | st.write(query) 15 | 16 | chat_completion = client.chat.completions.create( 17 | messages=[ 18 | {"role": "user","content": "You are a helpful assistant and provides structured answers."}, 19 | {"role": "user", "content": query} 20 | ], 21 | model="Phi-3-mini-128k-cuda-int4-onnx", 22 | ) 23 | with st.chat_message("assistant"): 24 | st.write(chat_completion.choices[0].message.content) 25 | 26 | -------------------------------------------------------------------------------- /Samples/1. Basic/AI_Toolkit_basic.py: -------------------------------------------------------------------------------- 1 | from openai import OpenAI 2 | 3 | client = OpenAI( 4 | base_url="http://127.0.0.1:5272/v1/", 5 | api_key="xyz" # required by API but not used 6 | ) 7 | 8 | chat_completion = client.chat.completions.create( 9 | messages=[ 10 | { 11 | "role": "user","content": "What is the capital of India?", 12 | } 13 | ], 14 | model="Phi-3-mini-128k-cuda-int4-onnx", 15 | ) 16 | 17 | print(chat_completion.choices[0].message.content) -------------------------------------------------------------------------------- /Samples/1. Basic/AI_Toolkit_console.py: -------------------------------------------------------------------------------- 1 | #Tutorial to connect with the VS Code AI Toolkit 2 | 3 | # Import the OpenAI class from the openai module 4 | from openai import OpenAI 5 | 6 | # Get the user input 7 | query=input("Enter query:") 8 | 9 | # Create an instance of the OpenAI class 10 | client = OpenAI( 11 | base_url="http://127.0.0.1:5272/v1/", 12 | api_key="xyz" # not used 13 | ) 14 | 15 | # Call the chat completions endpoint include the prompt and model 16 | chat_completion = client.chat.completions.create( 17 | messages=[ 18 | {"role": "user","content": "You are a helpful assistant and provides structured answers."}, 19 | {"role": "user", "content": query} 20 | ], 21 | model="Phi-3-mini-128k-cuda-int4-onnx", 22 | ) 23 | 24 | # Print the response 25 | print(chat_completion.choices[0].message.content) 26 | -------------------------------------------------------------------------------- /Samples/2. RAG/Embeddings/embeddings.py: -------------------------------------------------------------------------------- 1 | import spacy 2 | import pandas as pd 3 | 4 | # Load a spaCy language model 5 | #instead of OpenAI Text-embedding-ada-002 6 | nlp = spacy.load("en_core_web_md") #python -m spacy download en_core_web_md. 7 | 8 | with open("paragraph.txt", "r", encoding="utf-8") as file: 9 | paragraph = file.read() 10 | 11 | # Process the paragraph using spaCy 12 | doc = nlp(paragraph) 13 | 14 | # store vector representations 15 | vectors = [] 16 | 17 | # Iterate through the words in the paragraph and append their vectors to the list 18 | for token in doc: 19 | vectors.append(token.vector) 20 | 21 | # Create a DataFrame to store the vectors 22 | df = pd.DataFrame(vectors) 23 | 24 | # Save the vectors to a CSV file 25 | df.to_csv('embeddings.csv', index=False) 26 | 27 | ''' 28 | 29 | # paragraph = """ 30 | AI Toolkit for Visual Studio Code is an extension to help developers and AI engineers to easily build AI apps and agents through developing and testing with generative AI models locally or in the cloud. 31 | AI Toolkit supports most genAI models on the market. 32 | 33 | AI engineers can use AI Toolkit to discover and try popular AI models easily. 34 | 35 | The playground supports attachments, web search and thinking mode allowing for more interactive experimentation. 36 | They can run multiple prompts in batch mode and evaluate the prompts in a dataset to AI models using popular evaluators. 37 | Additionally, AI engineers can fine-tune and deploy AI models 38 | 39 | ''' -------------------------------------------------------------------------------- /Samples/2. RAG/Embeddings/paragraph.txt: -------------------------------------------------------------------------------- 1 | AI Toolkit for Visual Studio Code is an extension to help developers and AI engineers to easily build AI apps and agents through developing and testing with generative AI models locally or in the cloud. AI Toolkit supports most genAI models on the market. 2 | 3 | AI engineers can use AI Toolkit to discover and try popular AI models easily. The playground supports attachments, web search and thinking mode allowing for more interactive experimentation. They can run multiple prompts in batch mode and evaluate the prompts in a dataset to AI models using popular evaluators. Additionally, AI engineers can fine-tune and deploy AI models -------------------------------------------------------------------------------- /Samples/2. RAG/RAG_GithubModel/ai-toolkit/2f116a22-ece6-433b-9427-efbb2af4f4e2/header.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/RAG_GithubModel/ai-toolkit/2f116a22-ece6-433b-9427-efbb2af4f4e2/header.bin -------------------------------------------------------------------------------- /Samples/2. RAG/RAG_GithubModel/ai-toolkit/2f116a22-ece6-433b-9427-efbb2af4f4e2/length.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/RAG_GithubModel/ai-toolkit/2f116a22-ece6-433b-9427-efbb2af4f4e2/length.bin -------------------------------------------------------------------------------- /Samples/2. RAG/RAG_GithubModel/ai-toolkit/2f116a22-ece6-433b-9427-efbb2af4f4e2/link_lists.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/RAG_GithubModel/ai-toolkit/2f116a22-ece6-433b-9427-efbb2af4f4e2/link_lists.bin -------------------------------------------------------------------------------- /Samples/2. RAG/RAG_GithubModel/ai-toolkit/chroma.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/RAG_GithubModel/ai-toolkit/chroma.sqlite3 -------------------------------------------------------------------------------- /Samples/2. RAG/RAG_GithubModel/cloud_embedding_model.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "pip install openai" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "pip install dotenv" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": 24, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "import os\n", 28 | "from openai import OpenAI" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "execution_count": 25, 34 | "metadata": {}, 35 | "outputs": [ 36 | { 37 | "data": { 38 | "text/plain": [ 39 | "True" 40 | ] 41 | }, 42 | "execution_count": 25, 43 | "metadata": {}, 44 | "output_type": "execute_result" 45 | } 46 | ], 47 | "source": [ 48 | "from dotenv import load_dotenv\n", 49 | "\n", 50 | "load_dotenv()" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 26, 56 | "metadata": {}, 57 | "outputs": [], 58 | "source": [ 59 | "token = os.getenv('GITHUB_TOKEN')\n", 60 | "endpoint = \"https://models.inference.ai.azure.com\"\n", 61 | "model_name = \"text-embedding-3-small\"" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": null, 67 | "metadata": {}, 68 | "outputs": [], 69 | "source": [ 70 | "print(token)" 71 | ] 72 | }, 73 | { 74 | "cell_type": "code", 75 | "execution_count": 27, 76 | "metadata": {}, 77 | "outputs": [], 78 | "source": [ 79 | "client = OpenAI(\n", 80 | " base_url=endpoint,\n", 81 | " api_key=token,\n", 82 | ")\n" 83 | ] 84 | }, 85 | { 86 | "cell_type": "code", 87 | "execution_count": 28, 88 | "metadata": {}, 89 | "outputs": [ 90 | { 91 | "name": "stdout", 92 | "output_type": "stream", 93 | "text": [ 94 | "data[0]: length=1536, [-0.00721184303984046, 0.007491494063287973, ..., 0.01611734740436077, -0.004887983202934265]\n", 95 | "data[1]: length=1536, [-0.003025691257789731, 0.009231699630618095, ..., 0.029947662726044655, 0.020937401801347733]\n", 96 | "data[2]: length=1536, [-0.013795719482004642, 0.031857650727033615, ..., 0.017506178468465805, 0.0226223636418581]\n", 97 | "Usage(prompt_tokens=6, total_tokens=6)\n" 98 | ] 99 | } 100 | ], 101 | "source": [ 102 | "#Input a fixed list of phrases to get their embeddings\n", 103 | "\n", 104 | "response = client.embeddings.create(\n", 105 | " input=[\"first phrase\", \"second phrase\", \"third phrase\"],\n", 106 | " model=model_name,\n", 107 | ")\n", 108 | "\n", 109 | "for item in response.data:\n", 110 | " length = len(item.embedding)\n", 111 | " print(\n", 112 | " f\"data[{item.index}]: length={length}, \"\n", 113 | " f\"[{item.embedding[0]}, {item.embedding[1]}, \"\n", 114 | " f\"..., {item.embedding[length-2]}, {item.embedding[length-1]}]\"\n", 115 | " )\n", 116 | "print(response.usage)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": {}, 122 | "source": [ 123 | "To extract data from PDF" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": null, 129 | "metadata": {}, 130 | "outputs": [], 131 | "source": [ 132 | "pip install chromadb" 133 | ] 134 | }, 135 | { 136 | "cell_type": "code", 137 | "execution_count": null, 138 | "metadata": {}, 139 | "outputs": [], 140 | "source": [ 141 | "pip install pymupdf" 142 | ] 143 | }, 144 | { 145 | "cell_type": "code", 146 | "execution_count": null, 147 | "metadata": {}, 148 | "outputs": [], 149 | "source": [ 150 | "pip install langchain-openai" 151 | ] 152 | }, 153 | { 154 | "cell_type": "code", 155 | "execution_count": null, 156 | "metadata": {}, 157 | "outputs": [], 158 | "source": [ 159 | "pip install langchain-community" 160 | ] 161 | }, 162 | { 163 | "cell_type": "code", 164 | "execution_count": 30, 165 | "metadata": {}, 166 | "outputs": [], 167 | "source": [ 168 | "from langchain_community.document_loaders import DirectoryLoader,PyMuPDFLoader\n", 169 | "from langchain_community.vectorstores import Chroma\n", 170 | "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", 171 | "from langchain_openai import OpenAIEmbeddings" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": 31, 177 | "metadata": {}, 178 | "outputs": [], 179 | "source": [ 180 | "dir=\"docs/\"" 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": 33, 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "#Loading the documents from the directory\n", 190 | "\n", 191 | "def load_docs(dir):\n", 192 | " loader=DirectoryLoader(dir,loader_cls=PyMuPDFLoader,use_multithreading=True,max_concurrency=128,show_progress=True,silent_errors=True)\n", 193 | " documents=loader.load()\n", 194 | " return documents" 195 | ] 196 | }, 197 | { 198 | "cell_type": "code", 199 | "execution_count": 32, 200 | "metadata": {}, 201 | "outputs": [], 202 | "source": [ 203 | "#Splitting the documents into chunks\n", 204 | "\n", 205 | "def split_docs(documents,chunk_size=1000,chunk_overlap=100):\n", 206 | " text_splitter=RecursiveCharacterTextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap)\n", 207 | " docs=text_splitter.split_documents(documents)\n", 208 | " return docs" 209 | ] 210 | }, 211 | { 212 | "cell_type": "code", 213 | "execution_count": 34, 214 | "metadata": {}, 215 | "outputs": [ 216 | { 217 | "name": "stderr", 218 | "output_type": "stream", 219 | "text": [ 220 | "100%|██████████| 1/1 [00:00<00:00, 30.30it/s]\n" 221 | ] 222 | }, 223 | { 224 | "data": { 225 | "text/plain": [ 226 | "6" 227 | ] 228 | }, 229 | "execution_count": 34, 230 | "metadata": {}, 231 | "output_type": "execute_result" 232 | } 233 | ], 234 | "source": [ 235 | "documents=load_docs(dir)\n", 236 | "len(documents)" 237 | ] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "execution_count": 35, 242 | "metadata": {}, 243 | "outputs": [ 244 | { 245 | "name": "stdout", 246 | "output_type": "stream", 247 | "text": [ 248 | "12\n" 249 | ] 250 | } 251 | ], 252 | "source": [ 253 | "doc=split_docs(documents)\n", 254 | "print(len(doc))" 255 | ] 256 | }, 257 | { 258 | "cell_type": "code", 259 | "execution_count": 36, 260 | "metadata": {}, 261 | "outputs": [], 262 | "source": [ 263 | "#langchain client OpenAIEmbeddings\n", 264 | "\n", 265 | "clientopen = OpenAIEmbeddings(\n", 266 | " model=\"text-embedding-3-large\",\n", 267 | " base_url=endpoint,\n", 268 | " api_key=token,\n", 269 | ")" 270 | ] 271 | }, 272 | { 273 | "cell_type": "code", 274 | "execution_count": 37, 275 | "metadata": {}, 276 | "outputs": [], 277 | "source": [ 278 | "save_to=Chroma.from_documents(documents=doc,embedding=clientopen,persist_directory='./ai-toolkit')" 279 | ] 280 | }, 281 | { 282 | "cell_type": "code", 283 | "execution_count": 38, 284 | "metadata": {}, 285 | "outputs": [], 286 | "source": [ 287 | "query=\"What is AI Toolkit?\"" 288 | ] 289 | }, 290 | { 291 | "cell_type": "code", 292 | "execution_count": 39, 293 | "metadata": {}, 294 | "outputs": [ 295 | { 296 | "name": "stdout", 297 | "output_type": "stream", 298 | "text": [ 299 | "[Document(metadata={'author': 'Shreyan Fernandes', 'creationDate': \"D:20240708201313+05'30'\", 'creationdate': '2024-07-08T20:13:13+05:30', 'creator': 'Microsoft® Word 2019', 'file_path': 'docs\\\\AIToolkit.pdf', 'format': 'PDF 1.7', 'keywords': '', 'modDate': \"D:20240708201313+05'30'\", 'moddate': '2024-07-08T20:13:13+05:30', 'page': 0, 'producer': 'Microsoft® Word 2019', 'source': 'docs\\\\AIToolkit.pdf', 'subject': '', 'title': '', 'total_pages': 6, 'trapped': ''}, page_content='The AI Toolkit for Visual Studio Code (VS Code) was previously known as Windows AI Studio. The \\nextension has been renamed to reflect the focus on enabling AI development in VS Code across \\nplatforms. \\n \\nThe AI Toolkit for VS Code (AI Toolkit) is a VS Code extension that enables you to: \\n \\nDownload and run AI models locally. The AI Toolkit provides out-of-the-box access to highly \\noptimized models for the following platforms and hardware: \\n• \\nWindows 11 running with DirectML acceleration \\n• \\nWindows 11 running directly on the CPU \\n• \\nLinux with NVIDIA acceleration \\n• \\nLinux running directly on the CPU \\n• \\nTest models in an intuitive playground or in your application with a REST API. \\nFine-tune your AI model - locally or in the cloud (on a virtual machine) - to create new skills, improve \\nreliability of responses, set the tone and format of the response. The AI Toolkit provides a guided \\nwalkthrough to fine-tune popular small-language models (SLMs) - like Phi-3 and Mistral.'), Document(metadata={'author': 'Shreyan Fernandes', 'creationDate': \"D:20240708201313+05'30'\", 'creationdate': '2024-07-08T20:13:13+05:30', 'creator': 'Microsoft® Word 2019', 'file_path': 'docs\\\\AIToolkit.pdf', 'format': 'PDF 1.7', 'keywords': '', 'modDate': \"D:20240708201313+05'30'\", 'moddate': '2024-07-08T20:13:13+05:30', 'page': 0, 'producer': 'Microsoft® Word 2019', 'source': 'docs\\\\AIToolkit.pdf', 'subject': '', 'title': '', 'total_pages': 6, 'trapped': ''}, page_content='The AI Toolkit for Visual Studio Code (VS Code) was previously known as Windows AI Studio. The \\nextension has been renamed to reflect the focus on enabling AI development in VS Code across \\nplatforms. \\n \\nThe AI Toolkit for VS Code (AI Toolkit) is a VS Code extension that enables you to: \\n \\nDownload and run AI models locally. The AI Toolkit provides out-of-the-box access to highly \\noptimized models for the following platforms and hardware: \\n• \\nWindows 11 running with DirectML acceleration \\n• \\nWindows 11 running directly on the CPU \\n• \\nLinux with NVIDIA acceleration \\n• \\nLinux running directly on the CPU \\n• \\nTest models in an intuitive playground or in your application with a REST API. \\nFine-tune your AI model - locally or in the cloud (on a virtual machine) - to create new skills, improve \\nreliability of responses, set the tone and format of the response. The AI Toolkit provides a guided \\nwalkthrough to fine-tune popular small-language models (SLMs) - like Phi-3 and Mistral.'), Document(metadata={'author': 'Shreyan Fernandes', 'creationDate': \"D:20240708201313+05'30'\", 'creationdate': '2024-07-08T20:13:13+05:30', 'creator': 'Microsoft® Word 2019', 'file_path': 'docs\\\\AIToolkit.pdf', 'format': 'PDF 1.7', 'keywords': '', 'modDate': \"D:20240708201313+05'30'\", 'moddate': '2024-07-08T20:13:13+05:30', 'page': 0, 'producer': 'Microsoft® Word 2019', 'source': 'docs\\\\AIToolkit.pdf', 'subject': '', 'title': '', 'total_pages': 6, 'trapped': ''}, page_content='The AI Toolkit for Visual Studio Code (VS Code) was previously known as Windows AI Studio. The \\nextension has been renamed to reflect the focus on enabling AI development in VS Code across \\nplatforms. \\n \\nThe AI Toolkit for VS Code (AI Toolkit) is a VS Code extension that enables you to: \\n \\nDownload and run AI models locally. The AI Toolkit provides out-of-the-box access to highly \\noptimized models for the following platforms and hardware: \\n• \\nWindows 11 running with DirectML acceleration \\n• \\nWindows 11 running directly on the CPU \\n• \\nLinux with NVIDIA acceleration \\n• \\nLinux running directly on the CPU \\n• \\nTest models in an intuitive playground or in your application with a REST API. \\nFine-tune your AI model - locally or in the cloud (on a virtual machine) - to create new skills, improve \\nreliability of responses, set the tone and format of the response. The AI Toolkit provides a guided \\nwalkthrough to fine-tune popular small-language models (SLMs) - like Phi-3 and Mistral.'), Document(metadata={'author': 'Shreyan Fernandes', 'creationDate': \"D:20240708201313+05'30'\", 'creationdate': '2024-07-08T20:13:13+05:30', 'creator': 'Microsoft® Word 2019', 'file_path': 'docs\\\\AIToolkit.pdf', 'format': 'PDF 1.7', 'keywords': '', 'modDate': \"D:20240708201313+05'30'\", 'moddate': '2024-07-08T20:13:13+05:30', 'page': 0, 'producer': 'Microsoft® Word 2019', 'source': 'docs\\\\AIToolkit.pdf', 'subject': '', 'title': '', 'total_pages': 6, 'trapped': ''}, page_content=\"walkthrough to fine-tune popular small-language models (SLMs) - like Phi-3 and Mistral. \\nDeploy your AI feature either to the cloud or with an application that runs on a device. \\n \\nThe AI Toolkit for VS Code (AI Toolkit) is a VS Code extension that enables you to download, test, fine-\\ntune, and deploy AI models with your apps or the cloud. For more information, see the AI Toolkit \\noverview. \\n \\nIn this article, you'll learn how to: \\n \\n• \\nInstall the AI Toolkit for VS Code \\n• \\nDownload a model from the catalog \\n• \\nRun the model locally using the playground \\n• \\nIntegrate an AI model into your application using REST or the ONNX Runtime \\nPrerequisites \\nVS Code must be installed. For more information, see Download VS Code and Getting started with VS \\nCode. \\nInstall \\nThe AI Toolkit is available in the Visual Studio Marketplace and can be installed like any other VS \\nCode extension. If you're unfamiliar with installing VS Code extensions, follow these steps: \\n \\n•\")]\n", 300 | "The AI Toolkit for Visual Studio Code (VS Code) was previously known as Windows AI Studio. The \n", 301 | "extension has been renamed to reflect the focus on enabling AI development in VS Code across \n", 302 | "platforms. \n", 303 | " \n", 304 | "The AI Toolkit for VS Code (AI Toolkit) is a VS Code extension that enables you to: \n", 305 | " \n", 306 | "Download and run AI models locally. The AI Toolkit provides out-of-the-box access to highly \n", 307 | "optimized models for the following platforms and hardware: \n", 308 | "• \n", 309 | "Windows 11 running with DirectML acceleration \n", 310 | "• \n", 311 | "Windows 11 running directly on the CPU \n", 312 | "• \n", 313 | "Linux with NVIDIA acceleration \n", 314 | "• \n", 315 | "Linux running directly on the CPU \n", 316 | "• \n", 317 | "Test models in an intuitive playground or in your application with a REST API. \n", 318 | "Fine-tune your AI model - locally or in the cloud (on a virtual machine) - to create new skills, improve \n", 319 | "reliability of responses, set the tone and format of the response. The AI Toolkit provides a guided \n", 320 | "walkthrough to fine-tune popular small-language models (SLMs) - like Phi-3 and Mistral.\n" 321 | ] 322 | } 323 | ], 324 | "source": [ 325 | "db1=Chroma(persist_directory='./ai-toolkit',embedding_function=clientopen)\n", 326 | "results=db1.similarity_search(query)\n", 327 | "print(results)\n", 328 | "print(results[0].page_content)" 329 | ] 330 | } 331 | ], 332 | "metadata": { 333 | "kernelspec": { 334 | "display_name": ".venv", 335 | "language": "python", 336 | "name": "python3" 337 | }, 338 | "language_info": { 339 | "codemirror_mode": { 340 | "name": "ipython", 341 | "version": 3 342 | }, 343 | "file_extension": ".py", 344 | "mimetype": "text/x-python", 345 | "name": "python", 346 | "nbconvert_exporter": "python", 347 | "pygments_lexer": "ipython3", 348 | "version": "3.11.9" 349 | } 350 | }, 351 | "nbformat": 4, 352 | "nbformat_minor": 2 353 | } 354 | -------------------------------------------------------------------------------- /Samples/2. RAG/RAG_GithubModel/cloud_model_rag.py: -------------------------------------------------------------------------------- 1 | from langchain_openai import ChatOpenAI 2 | from langchain_community.vectorstores import Chroma 3 | from langchain_community.embeddings import SentenceTransformerEmbeddings 4 | from langchain_core.output_parsers import StrOutputParser 5 | from langchain_core.prompts import ChatPromptTemplate 6 | from langchain_core.runnables import RunnableParallel, RunnablePassthrough 7 | from dotenv import load_dotenv 8 | import os 9 | from langchain_openai import OpenAIEmbeddings 10 | load_dotenv() 11 | token = os.getenv('GITHUB_TOKEN') 12 | endpoint = "https://models.inference.ai.azure.com" 13 | model_name = "text-embedding-3-small" 14 | 15 | #embeddings=SentenceTransformerEmbeddings(model_name='all-MiniLM-L6-v2') 16 | 17 | embeddings = OpenAIEmbeddings( 18 | model="text-embedding-3-large", 19 | base_url=endpoint, 20 | api_key=token, 21 | ) 22 | 23 | model=ChatOpenAI( 24 | base_url="http://127.0.0.1:5272/v1/", 25 | api_key="ai-toolkit", 26 | model="Phi-3-mini-128k-directml-int4-awq-block-128-onnx", 27 | temperature=0.7 28 | ) 29 | 30 | load_db=Chroma(persist_directory='./ai-toolkit',embedding_function=embeddings) 31 | retriever=load_db.as_retriever(search_kwargs={'k':3}) 32 | 33 | template = """ You are a specialized AI assistant for the Microsoft Visual Studio Code AI Toolkit.\n 34 | Your responses should be strictly relevant to this product and the user's query. \n 35 | Avoid providing information that is not directly related to the toolkit. 36 | Maintain a professional tone and ensure your responses are accurate and helpful. 37 | Strictly adhere to the user's question and provide relevant information. 38 | If you do not know the answer then respond "I dont know".Do not refer to your knowledge base. 39 | {context} 40 | Question: 41 | {question} 42 | """ 43 | 44 | prompt = ChatPromptTemplate.from_template(template) 45 | output_parser = StrOutputParser() 46 | 47 | #RunnableParallel object is created to run multiple tasks in parallel 48 | 49 | setup_and_retrieval = RunnableParallel( 50 | {"context": retriever, "question": RunnablePassthrough()} 51 | ) 52 | 53 | #Represents the entire processing pipeline, where each component processes the input and passes the result to the next component. 54 | chain = setup_and_retrieval | prompt | model | output_parser 55 | query=input("Enter your query:") 56 | resp=chain.invoke(query) 57 | print(resp) -------------------------------------------------------------------------------- /Samples/2. RAG/RAG_GithubModel/docs/AIToolkit.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/RAG_GithubModel/docs/AIToolkit.pdf -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/Console.py: -------------------------------------------------------------------------------- 1 | from langchain_openai import ChatOpenAI 2 | from langchain_community.vectorstores import Chroma 3 | from langchain_community.embeddings import SentenceTransformerEmbeddings 4 | from langchain_core.output_parsers import StrOutputParser 5 | from langchain_core.prompts import ChatPromptTemplate 6 | from langchain_core.runnables import RunnableParallel, RunnablePassthrough 7 | 8 | 9 | embeddings=SentenceTransformerEmbeddings(model_name='all-MiniLM-L6-v2') 10 | 11 | model=ChatOpenAI( 12 | base_url="http://127.0.0.1:5272/v1/", 13 | api_key="ai-toolkit", 14 | model="Phi-3-mini-128k-directml-int4-awq-block-128-onnx", 15 | temperature=0.7 16 | ) 17 | 18 | load_db=Chroma(persist_directory='./ai-toolkit',embedding_function=embeddings) 19 | retriever=load_db.as_retriever(search_kwargs={'k':3}) 20 | 21 | template = """ You are a specialized AI assistant for the Microsoft Visual Studio Code AI Toolkit.\n 22 | Your responses should be strictly relevant to this product and the user's query. \n 23 | Avoid providing information that is not directly related to the toolkit. 24 | Maintain a professional tone and ensure your responses are accurate and helpful. 25 | Strictly adhere to the user's question and provide relevant information. 26 | If you do not know the answer then respond "I dont know".Do not refer to your knowledge base. 27 | {context} 28 | Question: 29 | {question} 30 | """ 31 | 32 | prompt = ChatPromptTemplate.from_template(template) 33 | output_parser = StrOutputParser() 34 | 35 | setup_and_retrieval = RunnableParallel( 36 | {"context": retriever, "question": RunnablePassthrough()} 37 | ) 38 | chain = setup_and_retrieval | prompt | model | output_parser 39 | 40 | resp=chain.invoke("What is Fine tuning") 41 | print(resp) -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/ai-toolkit/bafc4831-b7f7-461f-a499-08a5a0245047/header.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/Streamlit_PDF_OfflineModel/ai-toolkit/bafc4831-b7f7-461f-a499-08a5a0245047/header.bin -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/ai-toolkit/bafc4831-b7f7-461f-a499-08a5a0245047/length.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/Streamlit_PDF_OfflineModel/ai-toolkit/bafc4831-b7f7-461f-a499-08a5a0245047/length.bin -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/ai-toolkit/bafc4831-b7f7-461f-a499-08a5a0245047/link_lists.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/Streamlit_PDF_OfflineModel/ai-toolkit/bafc4831-b7f7-461f-a499-08a5a0245047/link_lists.bin -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/ai-toolkit/chroma.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/Streamlit_PDF_OfflineModel/ai-toolkit/chroma.sqlite3 -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/app.py: -------------------------------------------------------------------------------- 1 | # Tutorial for creating baseline RAG chatbot using AI Toolkit and Streamlit 2 | 3 | # Importing required libraries 4 | import streamlit as st 5 | from langchain_openai import ChatOpenAI 6 | from langchain_community.vectorstores import Chroma 7 | from langchain_community.embeddings import SentenceTransformerEmbeddings 8 | from langchain_core.output_parsers import StrOutputParser 9 | from langchain_core.prompts import ChatPromptTemplate 10 | from langchain_core.runnables import RunnableParallel, RunnablePassthrough 11 | 12 | # Initializing the required components for RAG 13 | embeddings = SentenceTransformerEmbeddings(model_name='all-MiniLM-L6-v2') 14 | 15 | model = ChatOpenAI( 16 | base_url="http://127.0.0.1:5272/v1/", 17 | api_key="ai-toolkit", 18 | model="Phi-3-mini-128k-directml-int4-awq-block-128-onnx", 19 | temperature=0.7 20 | ) 21 | 22 | # Loading the knowledge base 23 | load_db = Chroma(persist_directory='./ai-toolkit', embedding_function=embeddings) 24 | retriever = load_db.as_retriever(search_kwargs={'k': 3}) 25 | 26 | # Defining the template 27 | template = """ You are a specialized AI assistant for the Microsoft Visual Studio Code AI Toolkit.\n 28 | Your responses should be strictly relevant to this product and the user's query. \n 29 | Avoid providing information that is not directly related to the toolkit. 30 | Maintain a professional tone and ensure your responses are accurate and helpful. 31 | Strictly adhere to the user's question and provide relevant information. 32 | If you do not know the answer then respond "I dont know".Do not refer to your knowledge base. 33 | {context} 34 | Question: 35 | {question} 36 | """ 37 | 38 | # Defining the pipeline 39 | prompt = ChatPromptTemplate.from_template(template) 40 | output_parser = StrOutputParser() 41 | 42 | # Defining the pipeline 43 | setup_and_retrieval = RunnableParallel( 44 | {"context": retriever, "question": RunnablePassthrough()} 45 | ) 46 | chain = setup_and_retrieval | prompt | model | output_parser 47 | 48 | # Streamlit app 49 | st.title("AI Toolkit Chatbot") 50 | st.write("Ask me anything about the Microsoft Visual Studio Code AI Toolkit.") 51 | 52 | # user session 53 | if 'messages' not in st.session_state: 54 | st.session_state.messages = [] 55 | 56 | for message in st.session_state.messages: 57 | with st.chat_message(message["role"]): 58 | st.markdown(message["content"]) 59 | 60 | if user_input := st.chat_input("Your question:"): 61 | st.session_state.messages.append({"role": "user", "content": user_input}) 62 | with st.chat_message("user"): 63 | st.markdown(user_input) 64 | 65 | response = chain.invoke(user_input) 66 | st.session_state.messages.append({"role": "assistant", "content": response}) 67 | with st.chat_message("assistant"): 68 | st.markdown(response) -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/dbmaker.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from langchain_community.vectorstores import Chroma\n", 10 | "from langchain_community.embeddings import HuggingFaceEmbeddings\n", 11 | "from langchain_community.document_loaders import DirectoryLoader,PyMuPDFLoader\n", 12 | "from langchain.text_splitter import RecursiveCharacterTextSplitter" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 2, 18 | "metadata": {}, 19 | "outputs": [ 20 | { 21 | "name": "stderr", 22 | "output_type": "stream", 23 | "text": [ 24 | "c:\\Users\\shrey\\Desktop\\Work\\AI Toolkit\\Code\\RAG\\.venv\\Lib\\site-packages\\langchain_core\\_api\\deprecation.py:141: LangChainDeprecationWarning: The class `HuggingFaceEmbeddings` was deprecated in LangChain 0.2.2 and will be removed in 0.3.0. An updated version of the class exists in the langchain-huggingface package and should be used instead. To use it run `pip install -U langchain-huggingface` and import as `from langchain_huggingface import HuggingFaceEmbeddings`.\n", 25 | " warn_deprecated(\n", 26 | "c:\\Users\\shrey\\Desktop\\Work\\AI Toolkit\\Code\\RAG\\.venv\\Lib\\site-packages\\sentence_transformers\\cross_encoder\\CrossEncoder.py:11: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", 27 | " from tqdm.autonotebook import tqdm, trange\n" 28 | ] 29 | } 30 | ], 31 | "source": [ 32 | "#Directory of the PDF Files\n", 33 | "dir = 'docs/'\n", 34 | "\n", 35 | "# OS Embedding model from Huggingface\n", 36 | "embeddings = HuggingFaceEmbeddings(model_name='all-MiniLM-L6-v2')" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": 3, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "def load_docs(dir):\n", 46 | " loader=DirectoryLoader(dir,loader_cls=PyMuPDFLoader,use_multithreading=True,max_concurrency=128,show_progress=True,silent_errors=True)\n", 47 | " documents=loader.load()\n", 48 | " return documents" 49 | ] 50 | }, 51 | { 52 | "cell_type": "code", 53 | "execution_count": 4, 54 | "metadata": {}, 55 | "outputs": [], 56 | "source": [ 57 | "#Splitting the documents into chunks\n", 58 | "def split_docs(documents,chunk_size=1000,chunk_overlap=100):\n", 59 | " text_splitter=RecursiveCharacterTextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap)\n", 60 | " docs=text_splitter.split_documents(documents)\n", 61 | " return docs" 62 | ] 63 | }, 64 | { 65 | "cell_type": "code", 66 | "execution_count": 5, 67 | "metadata": {}, 68 | "outputs": [ 69 | { 70 | "name": "stderr", 71 | "output_type": "stream", 72 | "text": [ 73 | "100%|██████████| 2/2 [00:00<00:00, 3.91it/s]\n" 74 | ] 75 | }, 76 | { 77 | "data": { 78 | "text/plain": [ 79 | "14" 80 | ] 81 | }, 82 | "execution_count": 5, 83 | "metadata": {}, 84 | "output_type": "execute_result" 85 | } 86 | ], 87 | "source": [ 88 | "documents=load_docs(dir)\n", 89 | "len(documents)" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": 6, 95 | "metadata": {}, 96 | "outputs": [ 97 | { 98 | "name": "stdout", 99 | "output_type": "stream", 100 | "text": [ 101 | "27\n" 102 | ] 103 | } 104 | ], 105 | "source": [ 106 | "doc=split_docs(documents)\n", 107 | "print(len(doc))" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": 7, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "save_to=Chroma.from_documents(documents=doc,embedding=embeddings,persist_directory='./ai-toolkit')\n" 117 | ] 118 | }, 119 | { 120 | "cell_type": "code", 121 | "execution_count": 8, 122 | "metadata": {}, 123 | "outputs": [], 124 | "source": [ 125 | "query=\"What is Fine tuning\"" 126 | ] 127 | }, 128 | { 129 | "cell_type": "code", 130 | "execution_count": 9, 131 | "metadata": {}, 132 | "outputs": [ 133 | { 134 | "name": "stdout", 135 | "output_type": "stream", 136 | "text": [ 137 | "[Document(metadata={'author': '', 'creationDate': \"D:20240708144232+00'00'\", 'creator': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', 'file_path': 'docs\\\\Finetune.pdf', 'format': 'PDF 1.4', 'keywords': '', 'modDate': \"D:20240708144232+00'00'\", 'page': 1, 'producer': 'Skia/PDF m126', 'source': 'docs\\\\Finetune.pdf', 'subject': '', 'title': 'Fine-tune a model with the AI Toolkit for Visual Studio Code | Microsoft Learn', 'total_pages': 8, 'trapped': ''}, page_content='AI Toolkit: Validate Environment prerequisites.\\nIf your local device passes the validation checks, the Setup WSL Environment button\\nwill be enabled for you to select. This will install all the dependencies required to run\\nfine-tuning jobs.\\nIf your local computer does not have an Nvidia GPU device, it is possible to fine-tune on\\na cloud VM - both Windows and Linux - with an Nvidia GPU (if you have quota). In\\nAzure, you can fine-tune with the following VM series:\\nNCasT4_v3-series\\nNC A100 v4-series\\nND A100 v4-series\\nNCads H100 v5-series\\nNCv3-series\\nNVadsA10 v5-series\\nThe AI Toolkit uses a method called QLoRA, which combines quantization and low-rank\\nadaptation (LoRA) to fine-tune models with your own data. Learn more about QLoRA at\\nQLoRA: Efficient Finetuning of Quantized LLMs\\n.\\nTo start a new fine-tuning session using QLoRA, select the Model Fine-tuning item in AI\\nToolkit.\\nStart by entering a unique Project Name and a Project Location. A new folder with the'), Document(metadata={'author': '', 'creationDate': \"D:20240708144232+00'00'\", 'creator': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', 'file_path': 'docs\\\\Finetune.pdf', 'format': 'PDF 1.4', 'keywords': '', 'modDate': \"D:20240708144232+00'00'\", 'page': 5, 'producer': 'Skia/PDF m126', 'source': 'docs\\\\Finetune.pdf', 'subject': '', 'title': 'Fine-tune a model with the AI Toolkit for Visual Studio Code | Microsoft Learn', 'total_pages': 8, 'trapped': ''}, page_content='ﾉ\\nExpand table\\nStep 3: Execute fine-tuning job\\n7/8/24, 8:12 PM\\nFine-tune a model with the AI Toolkit for Visual Studio Code | Microsoft Learn\\nhttps://learn.microsoft.com/en-us/windows/ai/toolkit/toolkit-fine-tune\\n6/8'), Document(metadata={'author': '', 'creationDate': \"D:20240708144232+00'00'\", 'creator': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', 'file_path': 'docs\\\\Finetune.pdf', 'format': 'PDF 1.4', 'keywords': '', 'modDate': \"D:20240708144232+00'00'\", 'page': 2, 'producer': 'Skia/PDF m126', 'source': 'docs\\\\Finetune.pdf', 'subject': '', 'title': 'Fine-tune a model with the AI Toolkit for Visual Studio Code | Microsoft Learn', 'total_pages': 8, 'trapped': ''}, page_content=\"You'll then be prompted to configure your fine-tuning project settings. Ensure the Fine-\\ntune locally checkbox is ticked (in the future the VS Code extension will allow you to\\noffload fine-tuning to the cloud):\\nThere are two settings available in the Model inference section:\\nModel inference settings\\n7/8/24, 8:12 PM\\nFine-tune a model with the AI Toolkit for Visual Studio Code | Microsoft Learn\\nhttps://learn.microsoft.com/en-us/windows/ai/toolkit/toolkit-fine-tune\\n3/8\"), Document(metadata={'author': '', 'creationDate': \"D:20240708144232+00'00'\", 'creator': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36', 'file_path': 'docs\\\\Finetune.pdf', 'format': 'PDF 1.4', 'keywords': '', 'modDate': \"D:20240708144232+00'00'\", 'page': 3, 'producer': 'Skia/PDF m126', 'source': 'docs\\\\Finetune.pdf', 'subject': '', 'title': 'Fine-tune a model with the AI Toolkit for Visual Studio Code | Microsoft Learn', 'total_pages': 8, 'trapped': ''}, page_content='Source max\\nlength\\nThe maximum number of tokens per training sample.\\nPad to max\\nlength\\nAdd a PAD token to the training sample until the max number of tokens.\\nThe following settings are available in the Fine tune section to further configure the\\nfine-tuning process:\\nﾉ\\nExpand table\\nData settings\\nﾉ\\nExpand table\\nFine-tune settings\\nﾉ\\nExpand table\\n7/8/24, 8:12 PM\\nFine-tune a model with the AI Toolkit for Visual Studio Code | Microsoft Learn\\nhttps://learn.microsoft.com/en-us/windows/ai/toolkit/toolkit-fine-tune\\n4/8')]\n", 138 | "AI Toolkit: Validate Environment prerequisites.\n", 139 | "If your local device passes the validation checks, the Setup WSL Environment button\n", 140 | "will be enabled for you to select. This will install all the dependencies required to run\n", 141 | "fine-tuning jobs.\n", 142 | "If your local computer does not have an Nvidia GPU device, it is possible to fine-tune on\n", 143 | "a cloud VM - both Windows and Linux - with an Nvidia GPU (if you have quota). In\n", 144 | "Azure, you can fine-tune with the following VM series:\n", 145 | "NCasT4_v3-series\n", 146 | "NC A100 v4-series\n", 147 | "ND A100 v4-series\n", 148 | "NCads H100 v5-series\n", 149 | "NCv3-series\n", 150 | "NVadsA10 v5-series\n", 151 | "The AI Toolkit uses a method called QLoRA, which combines quantization and low-rank\n", 152 | "adaptation (LoRA) to fine-tune models with your own data. Learn more about QLoRA at\n", 153 | "QLoRA: Efficient Finetuning of Quantized LLMs\n", 154 | ".\n", 155 | "To start a new fine-tuning session using QLoRA, select the Model Fine-tuning item in AI\n", 156 | "Toolkit.\n", 157 | "Start by entering a unique Project Name and a Project Location. A new folder with the\n" 158 | ] 159 | } 160 | ], 161 | "source": [ 162 | "db1=Chroma(persist_directory='./ai-toolkit',embedding_function=embeddings)\n", 163 | "results=db1.similarity_search(query)\n", 164 | "print(results)\n", 165 | "print(results[0].page_content)" 166 | ] 167 | } 168 | ], 169 | "metadata": { 170 | "kernelspec": { 171 | "display_name": ".venv", 172 | "language": "python", 173 | "name": "python3" 174 | }, 175 | "language_info": { 176 | "codemirror_mode": { 177 | "name": "ipython", 178 | "version": 3 179 | }, 180 | "file_extension": ".py", 181 | "mimetype": "text/x-python", 182 | "name": "python", 183 | "nbconvert_exporter": "python", 184 | "pygments_lexer": "ipython3", 185 | "version": "3.11.9" 186 | } 187 | }, 188 | "nbformat": 4, 189 | "nbformat_minor": 2 190 | } 191 | -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/docs/AIToolkit.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/Streamlit_PDF_OfflineModel/docs/AIToolkit.pdf -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/docs/Finetune.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/2. RAG/Streamlit_PDF_OfflineModel/docs/Finetune.pdf -------------------------------------------------------------------------------- /Samples/2. RAG/Streamlit_PDF_OfflineModel/requirements.txt: -------------------------------------------------------------------------------- 1 | chromadb==0.5.3 2 | langchain==0.2.6 3 | langchain-community==0.2.6 4 | sentence-transformers 5 | pymupdf 6 | openai 7 | langchain-openai 8 | streamlit -------------------------------------------------------------------------------- /Samples/3.Multimodal/Basic Console/app.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import base64 3 | from openai import OpenAI 4 | 5 | client = OpenAI( 6 | base_url="http://127.0.0.1:5272/v1/", 7 | api_key="xyz" # required by API but not used 8 | ) 9 | 10 | st.title('Image Analyser Application') 11 | st.write('This is a simple Image Translator Application that uses the Phi-3 Vision model to analyse the image.') 12 | uploaded_file = st.file_uploader("Choose a file") 13 | 14 | 15 | def encode_image(uploaded_file): 16 | with open(uploaded_file, "rb") as uploaded_file: 17 | return base64.b64encode(uploaded_file.read()).decode("utf-8") 18 | 19 | base64_image = encode_image(uploaded_file) 20 | 21 | response = client.chat.completions.create( 22 | model="Phi-3-vision-128k-cpu-int4-rtn-block-32-acc-level-4-onnx", 23 | messages=[ 24 | { 25 | "role": "user", 26 | "content": [ 27 | { 28 | "type": "text", 29 | "text": "Analyse the following image and describe the image to the user", 30 | }, 31 | { 32 | "type": "image_url", 33 | "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}, 34 | }, 35 | ], 36 | } 37 | ], 38 | ) 39 | print(response.choices[0].message.content) 40 | st.write(response.choices[0].message.content) -------------------------------------------------------------------------------- /Samples/3.Multimodal/Basic Console/img.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/AI_Toolkit_Samples/e6386ab08d04fad9f425f49cee15baae2e6738af/Samples/3.Multimodal/Basic Console/img.jpg -------------------------------------------------------------------------------- /Samples/3.Multimodal/Recipe Generator/app.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import base64 3 | from openai import OpenAI 4 | 5 | client = OpenAI( 6 | base_url="http://127.0.0.1:5272/v1/", 7 | api_key="xyz" # required by API but not used 8 | ) 9 | 10 | st.title('Recipe Generator 🍔') 11 | st.write('This is a simple recipe generator application.Upload images of the Ingridients and get the recipe by Chef GenAI! 🧑‍🍳') 12 | uploaded_file = st.file_uploader("Choose a file") 13 | if uploaded_file is not None: 14 | st.image(uploaded_file, width=300) 15 | 16 | preference = st.sidebar.selectbox( 17 | "Choose your preference", 18 | ("Vegetarian", "Non-Vegetarian") 19 | ) 20 | 21 | cuisine = st.sidebar.selectbox( 22 | "Select for Cuisine", 23 | ("Indian","Chinese","French","Thai","Italian","Mexican","Japanese","American","Greek","Spanish") 24 | ) 25 | 26 | 27 | def encode_image(uploaded_file): 28 | """Encodes a Streamlit uploaded file into base64 format""" 29 | if uploaded_file is not None: 30 | content = uploaded_file.read() 31 | return base64.b64encode(content).decode("utf-8") 32 | else: 33 | return None 34 | 35 | base64_image = encode_image(uploaded_file) 36 | 37 | if st.button("Ask Chef GenAI!"): 38 | if base64_image: 39 | response = client.chat.completions.create( 40 | model="Phi-3-vision-128k-cpu-int4-rtn-block-32-acc-level-4-onnx", 41 | messages=[ 42 | { 43 | "role": "user", 44 | "content": [ 45 | { 46 | "type": "text", 47 | "text": f"STRICTLY use the ingredients in the image to generate a {preference} recipe and {cuisine} cuisine.", 48 | }, 49 | { 50 | "type": "image_url", 51 | "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}, 52 | }, 53 | ], 54 | } 55 | ], 56 | ) 57 | print(response.choices[0].message.content) 58 | st.write(response.choices[0].message.content) 59 | else: 60 | st.write("Please upload an image with any number of ingridients and instantly get a recipe.") 61 | 62 | 63 | -------------------------------------------------------------------------------- /Samples/3.Multimodal/requirements.txt: -------------------------------------------------------------------------------- 1 | openai 2 | streamlit --------------------------------------------------------------------------------