├── .devcontainer ├── java │ └── devcontainer.json └── python │ └── devcontainer.json ├── .gitignore ├── AgentMode ├── .github │ ├── copilot-instructions.md │ └── tsql │ │ ├── inner-loop │ │ ├── Build.ps1 │ │ ├── Publish.ps1 │ │ └── Test.ps1 │ │ ├── install │ │ ├── copy-tsqlt.dacpac-file.ps1 │ │ ├── create-mssql-instance-using-sqlcmd.ps1 │ │ ├── ensure-build-sql-templates-installed.ps1 │ │ ├── ensure-sqlcmd-installed.ps1 │ │ ├── ensure-sqlpackage-installed.ps1 │ │ ├── ensure-tsqlt-installed.ps1 │ │ ├── get-sql-connection-string.ps1 │ │ └── is-container-runtime-running.ps1 │ │ └── tsql-best-practices.md ├── ConfMSmini │ └── business-process-description.md ├── README.md └── empty-business-process-description.md ├── AzureSQLACSSamples ├── DataSet │ └── Reviews_small.csv └── src │ ├── AzureSQL_CogSearch_IntegratedVectorization.ipynb │ ├── example.env │ └── requirements.txt ├── AzureSQLDatabase ├── ContentModeration │ ├── contentSafety.sql │ └── languageAI.sql ├── LangChain │ ├── .env │ ├── README.md │ ├── database.sql │ ├── dbAzureOpenAI.ipynb │ ├── dbOpenAI.ipynb │ ├── odbcDriverInstallUbuntu.txt │ └── requirements.txt ├── Prompt-Based T-SQL Database Development │ ├── .DS_Store │ ├── End to End SQL Database Development - A Comprehensive guide with Interactive Prompts.pdf │ ├── bicycle_data_prompt.json │ ├── csv │ │ ├── bicycle_data.csv │ │ └── bicycle_data_bad.csv │ ├── dbproject │ │ ├── .DS_Store │ │ └── localhost │ │ │ └── localhost.sqlproj │ ├── deploy_bicycle_sales.ipynb │ ├── env.txt │ ├── readme.md │ └── tsql │ │ ├── create_database.sql │ │ ├── create_etl_errorlog_table.sql │ │ ├── create_etl_errorlog_usp.sql │ │ ├── create_etl_process_schema.sql │ │ ├── create_etl_processlog_table.sql │ │ ├── create_etl_processlog_usp.sql │ │ ├── create_prd_table.sql │ │ ├── create_procedure.sql │ │ ├── create_stage_schema.sql │ │ ├── create_stage_table.sql │ │ ├── create_view.sql │ │ ├── creates_prd_chema.sql │ │ ├── load_prd_data_table.sql │ │ └── load_staging_data_table.sql └── Vanna.ai │ ├── odbcDriverInstallUbuntu.txt │ └── vanna_and_sql.ipynb ├── AzureSQLFaiss ├── Azure SQL_and_Faiss_Index_Creation.ipynb └── Faiss_inference.ipynb ├── AzureSQLPromptFlowSamples ├── README.md └── src │ ├── requirements.txt │ └── sql-promptflow-demo │ ├── acs │ ├── .env_sample │ └── azure_congnitive_search_prepare.ipynb │ ├── batch_run_and_eval.py │ ├── configs │ ├── flow_config_sample.json │ └── key_config_local_sample.json │ ├── data │ ├── batch_run_data.jsonl │ ├── batch_run_data_v1.jsonl │ └── eval_sample.html │ ├── deploy.py │ ├── deployment │ ├── deployment_base.yaml │ ├── endpoint_base.yaml │ └── model_base.yaml │ ├── evaluation │ ├── .promptflow │ │ ├── flow.tools.json │ │ └── lkg_sources │ │ │ ├── aggregate_variants_results.py │ │ │ ├── concat_scores.py │ │ │ └── groundedness_score.jinja2 │ ├── aggregate_variants_results.py │ ├── concat_scores.py │ ├── flow.dag.sample.yaml │ ├── groundedness_score.jinja2 │ └── requirements.txt │ ├── promptflow │ ├── .promptflow │ │ ├── chat.detail.json │ │ ├── chat.output.json │ │ ├── flow.detail.json │ │ ├── flow.layout.json │ │ ├── flow.log │ │ └── flow.tools.json │ ├── __pycache__ │ │ ├── get_retrieved_documents.cpython-310.pyc │ │ ├── retrieve_customer.cpython-310.pyc │ │ ├── retrieve_orders.cpython-310.pyc │ │ ├── retrieve_products.cpython-310.pyc │ │ └── retrieve_products_stats.cpython-310.pyc │ ├── chat.jinja2 │ ├── flow.dag.sample.yaml │ ├── get_retrieved_documents.py │ ├── requirements.txt │ ├── retrieve_customer.py │ ├── retrieve_orders.py │ ├── retrieve_products.py │ ├── retrieve_products_stats.py │ └── single_run_data.jsonl │ ├── promptflow_v2 │ ├── chat.jinja2 │ ├── flow.dag.sample.yaml │ ├── get_customer.py │ ├── get_pastorders.py │ ├── get_product.py │ ├── get_produt_stats.py │ ├── get_retrieved_documents.py │ ├── requirements.txt │ ├── single_run_data.jsonl │ └── sql_query_store.py │ ├── run.py │ └── setup.py ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MssqlMcp ├── .editorconfig ├── .gitattributes ├── .github │ └── copilot-instructions.md ├── .gitignore ├── MssqlMcp.Tests │ ├── MssqlMcp.Tests.csproj │ └── UnitTests.cs ├── MssqlMcp.sln ├── MssqlMcp │ ├── DbOperationResult.cs │ ├── ISqlConnectionFactory.cs │ ├── MssqlMcp.csproj │ ├── Program.cs │ ├── SqlConnectionFactory.cs │ └── Tools │ │ ├── CreateTable.cs │ │ ├── DescribeTable.cs │ │ ├── DropTable.cs │ │ ├── InsertData.cs │ │ ├── ListTables.cs │ │ ├── ReadData.cs │ │ ├── Tools.cs │ │ └── UpdateData.cs └── README.md ├── README.md ├── SECURITY.md └── SUPPORT.md /.devcontainer/java/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "image": "mcr.microsoft.com/devcontainers/universal:2", 3 | "features": { 4 | "ghcr.io/devcontainers-contrib/features/maven-sdkman:2": {}, 5 | "ghcr.io/jlaundry/devcontainer-features/azure-functions-core-tools:1": {} 6 | }, 7 | "customizations": { 8 | "vscode": { 9 | "extensions": [ 10 | "vscjava.vscode-java-pack", 11 | "ms-azuretools.vscode-docker", 12 | "ms-mssql.mssql" 13 | ] 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.devcontainer/python/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "ghcr.io/devcontainers/features/python:1":{}, 3 | "features": { 4 | }, 5 | "customizations": { 6 | "vscode": { 7 | "extensions": [ 8 | "ms-python.python", 9 | "ms-toolsai.jupyter", 10 | "ms-mssql.mssql" 11 | ] 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## 4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 5 | 6 | .DS_Store 7 | .env 8 | .vscode 9 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/configs/flow_config.json 10 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/configs/key_config_local.json 11 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/flow.dag.yaml 12 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/flow.dag.yaml 13 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/.promptflow/ 14 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/__pycache__/ 15 | 16 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/deployment/deployment.yaml 17 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/deployment/endpoint.yaml 18 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/deployment/model.yaml 19 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/flow.dag.yaml 20 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/.promptflow/ 21 | AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/__pycache__/ 22 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/inner-loop/Build.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | param ( 5 | [string]$ProjectName 6 | ) 7 | 8 | # Run dotnet build 9 | dotnet build "$ProjectName/$ProjectName.sqlproj" 10 | if ($LASTEXITCODE -ne 0) { 11 | Write-Host "Failed to build the project. Please check the output for errors." 12 | exit 1 13 | } 14 | 15 | 16 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/inner-loop/Publish.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # -SourceFile e.g. ConfMS/bin/Debug/ConfMS.dacpac 5 | param ( 6 | [string]$ProjectName 7 | ) 8 | 9 | # Get the target connection string by running `get-sql-connection-string.ps1` 10 | $targetConnectionString = & ./.github/tsql/install/get-sql-connection-string.ps1 11 | 12 | # Ensure the connection string is wrapped in double quotes for SqlPackage.exe 13 | $quotedConnectionString = '"' + $targetConnectionString + '"' 14 | 15 | # Define the source file path 16 | $sourceFile = "./$ProjectName/bin/Debug/$ProjectName.dacpac" 17 | if (!(Test-Path $sourceFile)) { 18 | $sourceFile = "../$ProjectName/bin/Debug/$ProjectName.dacpac" 19 | } 20 | 21 | # Print the source file path 22 | Write-Host "Source file: $sourceFile" 23 | 24 | # Mask password in connection string for display 25 | $maskedConnectionString = $targetConnectionString -replace '(?i)(Password|Pwd)=([^;]*)', '$1=*****' 26 | 27 | # Print the publish command with masked password 28 | $maskedQuotedConnectionString = '"' + $maskedConnectionString + '"' 29 | $maskedPublishCommand = "$env:USERPROFILE\.dotnet\tools\SqlPackage.exe /Action:Publish /SourceFile:$sourceFile /TargetConnectionString:$maskedQuotedConnectionString /p:IncludeCompositeObjects=true" 30 | $maskedPublishCommand = $maskedPublishCommand -replace '\$', '`$' # Escape $ in the connection string 31 | Write-Host "Running command: $maskedPublishCommand" 32 | 33 | # Run the actual publish command (with real password) 34 | $publishCommand = "$env:USERPROFILE\.dotnet\tools\SqlPackage.exe /Action:Publish /SourceFile:$sourceFile /TargetConnectionString:$quotedConnectionString /p:IncludeCompositeObjects=true" 35 | $publishCommand = $publishCommand -replace '\$', '`$' # Escape $ in the connection string 36 | Invoke-Expression $publishCommand 37 | if ($LASTEXITCODE -ne 0) { 38 | Write-Host "Failed to publish the DACPAC. Please check the output for errors." 39 | exit 1 40 | } 41 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/inner-loop/Test.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # Run `sqlcmd query "EXEC tSQLt.RunAll` 5 | & $env:ProgramFiles\SqlCmd\sqlcmd.exe query "EXEC tSQLt.RunAll" 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/install/copy-tsqlt.dacpac-file.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # This powershell script takes a $projectPath 5 | # as a parameter and copies the tSQLt.dacpac file to the $projectPath\Tests 6 | # from the "$env:TEMP\tSQLt\tSQLtDacpacs" folder 7 | 8 | param ( 9 | [string]$projectPath 10 | ) 11 | 12 | # Define the temp folder for unzipped files 13 | $tempUnzipFolder = "$env:TEMP\tSQLt\tSQLtDacpacs" 14 | # Define the tSQLt class file name 15 | $tsqltDacPacFile = "tSQLt.2019.dacpac" 16 | # Define the tSQLt class file path 17 | $tsqltDacPacPath = "$tempUnzipFolder\$tsqltDacPacFile" 18 | # Check if the tSQLt class file exists 19 | if (Test-Path $tsqltDacPacPath) { 20 | # Copy the tSQLt.dacpac to the destination folder 21 | Copy-Item -Path $tsqltDacPacPath -Destination $projectPath\Tests -Force 22 | Write-Host "$tsqltDacPacFile file copied to $projectPath\Tests" 23 | } else { 24 | Write-Error "$tsqltDacPacFile file not found in $tempUnzipFolder" 25 | } 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/install/create-mssql-instance-using-sqlcmd.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # This script takes a parameter which is the context-name (passed into sqlcmd) 5 | # and creates a new SQL Server instance using the sqlcmd command line tool: 6 | # `sqlcmd create mssql --tag 2019-latest --context-name $ContextName` 7 | 8 | # If the context already exists, it will be removed first. 9 | 10 | param ( 11 | [string]$ProjectName 12 | ) 13 | 14 | # Run `sqlcmd config get-contexts`, it returns a lists of contexts, e.g. 15 | # 16 | # ``` 17 | # - mssql 18 | # - mssql2 19 | # - mssql3 20 | # ``` 21 | 22 | # Check if the context already exists 23 | $existingContexts = & $env:ProgramFiles\SqlCmd\sqlcmd.exe config get-contexts 24 | 25 | # Normalize context names by trimming and removing dashes 26 | $normalizedContexts = $existingContexts | ForEach-Object { $_.Trim().TrimStart('-').Trim() } 27 | $contextExists = $normalizedContexts | Where-Object { $_ -ieq $ProjectName } 28 | if ($contextExists) { 29 | # Run `sqlcmd use-context $ProjectName`, it sets the current context, which we'll then delete 30 | & $env:ProgramFiles\SqlCmd\sqlcmd.exe config use-context $ProjectName 31 | 32 | # Delete the context by running `sqlcmd delete --force --yes` 33 | & $env:ProgramFiles\SqlCmd\sqlcmd.exe delete --force --yes 34 | } 35 | 36 | # Create a new SQL Server instance using the sqlcmd command line tool 37 | & $env:ProgramFiles\SqlCmd\sqlcmd.exe create mssql --tag 2019-latest --accept-eula --user-database $ProjectName --context-name $ProjectName 38 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/install/ensure-build-sql-templates-installed.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # Use the dotnet CLI to see if Build.Sql.Templates are installed, if not, install them 5 | 6 | # Check if Build.Sql.Templates is installed 7 | $installed = dotnet new list | Select-String -Pattern "SQL Server Database Project" 8 | if ($installed -eq $null) { 9 | Write-Host "Build.Sql.Templates is not installed. Installing..." 10 | # Install Build.Sql.Templates 11 | dotnet new install Microsoft.Build.Sql.Templates 12 | } else { 13 | Write-Host "Build.Sql.Templates is already installed." 14 | } 15 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/install/ensure-sqlcmd-installed.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # Ensure sqlcmd is installed in C:\Program Files\SqlCmd\sqlcmd.exe 5 | 6 | # Test if 'sqlcmd' is available in `C:\Program Files\SqlCmd` 7 | 8 | $expectedPath = "$env:ProgramFiles\SqlCmd\sqlcmd.exe" 9 | 10 | if (-not (Test-Path $expectedPath)){ 11 | Write-Host "'sqlcmd' not found in $env:ProgramFiles. Attempting to install using winget..." 12 | 13 | # Check if winget is available 14 | if (-not (Get-Command winget -ErrorAction SilentlyContinue)) { 15 | Write-Error "'winget' is not installed. Please install winget manually and re-run this script." 16 | exit 1 17 | } 18 | 19 | # Install Microsoft SQL Server Command Line Tools (sqlcmd) using winget 20 | winget install sqlcmd -e --accept-source-agreements --accept-package-agreements 21 | 22 | # Verify installation 23 | if (-not (Get-Command sqlcmd -ErrorAction SilentlyContinue)) { 24 | Write-Error "'sqlcmd' installation failed. Please install manually." 25 | exit 1 26 | } else { 27 | Write-Host "'sqlcmd' installed successfully." 28 | } 29 | } else { 30 | Write-Host "'sqlcmd' is already installed at $expectedPath." 31 | 32 | } -------------------------------------------------------------------------------- /AgentMode/.github/tsql/install/ensure-sqlpackage-installed.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # Check to see if SqlPackage is installed by running: 5 | # `dotnet tool list --format json -g microsoft.sqlpackage` 6 | # if it is installed, the response will be in the form: 7 | # ``` 8 | # {"version":1,"data":[{"packageId":"microsoft.sqlpackage","version":"170.1.14","commands":["sqlpackage"]}]} 9 | # ``` 10 | 11 | # if not installed, install it by running: 12 | # `dotnet tool install -g microsoft.sqlpackage` 13 | 14 | # Check if SqlPackage is installed 15 | $installed = dotnet tool list --format json -g microsoft.sqlpackage | ConvertFrom-Json 16 | if ($installed.data.Count -gt 0) { 17 | Write-Host "SqlPackage is already installed." 18 | exit 0 19 | } 20 | 21 | # If SqlPackage is not installed, install it 22 | Write-Host "SqlPackage is not installed. Installing..." 23 | dotnet tool install -g microsoft.sqlpackage 24 | if ($LASTEXITCODE -ne 0) { 25 | Write-Host "Failed to install SqlPackage. Please check your .NET installation and try again." 26 | exit 1 27 | } 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/install/ensure-tsqlt-installed.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # Install tSQLt if it is not already installed 5 | 6 | # If tSQLt is not installed: 7 | # - Download from: https://tsqlt.org/download/tsqlt/ to a temp folder 8 | # - Unzip the downloaded file to a temp folder 9 | # - Execute the PrepareServer.sql using sqlcmd, this only has to be done once per server. 10 | 11 | # Define the URL for the tSQLt download 12 | $tsqltUrl = "https://tsqlt.org/download/tsqlt/" 13 | # Define the temp folder for downloads 14 | $tempFolder = "$env:TEMP" 15 | # Define the temp folder for unzipped files 16 | $tempUnzipFolder = "$env:TEMP\tSQLt" 17 | # Define the tSQLt zip file name 18 | $tsqltZipFile = "tSQLt.zip" 19 | # Define the tSQLt prepare server file name 20 | $tsqltPrepareServerFile = "PrepareServer.sql" 21 | 22 | # Define the SQLCMD command as an array for PowerShell invocation 23 | $sqlcmdExe = "$env:ProgramFiles\SqlCmd\sqlcmd.exe" 24 | $sqlcmdArgs = @("-i") 25 | 26 | # Check if tSQLt is already installed 27 | $checkTSQLtInstalled = & $sqlcmdExe -Q "SELECT COUNT(*) FROM sys.objects WHERE name = 'tSQLt' AND type = 'P'" -h -1 -W | Select-Object -First 1 28 | if ($checkTSQLtInstalled -eq 0) { 29 | Write-Host "tSQLt is not installed. Installing tSQLt..." 30 | 31 | # Create the temp folder if it doesn't exist 32 | if (-not (Test-Path $tempUnzipFolder)) { 33 | New-Item -ItemType Directory -Path $tempUnzipFolder 34 | } 35 | 36 | # Download the tSQLt zip file 37 | Invoke-WebRequest -Uri $tsqltUrl -OutFile "$tempFolder\$tsqltZipFile" 38 | 39 | # Unzip the downloaded file 40 | Expand-Archive -Force -Path "$tempFolder\$tsqltZipFile" -DestinationPath $tempUnzipFolder 41 | 42 | } else { 43 | Write-Host "tSQLt is already installed." 44 | } 45 | 46 | # Execute the PrepareServer.sql script 47 | $prepareServerPath = "$tempUnzipFolder\$tsqltPrepareServerFile" 48 | & $sqlcmdExe @($sqlcmdArgs + $prepareServerPath) 49 | Write-Host "tSQLt installation completed." 50 | 51 | # Delete the tsqlt zip file 52 | if (Test-Path "$tempFolder\$tsqltZipFile") { 53 | Remove-Item "$tempFolder\$tsqltZipFile" 54 | } -------------------------------------------------------------------------------- /AgentMode/.github/tsql/install/get-sql-connection-string.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # Return the ADO.Net SQL connection string by running `sqlcmd config connection-strings`, which will return 5 | # 6 | 7 | # Run `sqlcmd config connection-strings` to get the connection strings 8 | # and filter the output to get the ADO.Net connection string 9 | $adoNetConnectionString = & $env:ProgramFiles\SqlCmd\sqlcmd.exe config connection-strings | Select-String -Pattern "ADO.NET" | ForEach-Object { $_.ToString().Trim() } 10 | 11 | # Remove the "ADO.NET: " prefix from the connection string 12 | $adoNetConnectionString = $adoNetConnectionString -replace "ADO.NET: ", "" 13 | 14 | # Return the ADO.Net connection string 15 | $adoNetConnectionString 16 | 17 | 18 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/install/is-container-runtime-running.ps1: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. All rights reserved. 2 | # Licensed under the MIT license. 3 | 4 | # A powershell script for Windows to check if any desktop container runtime is 5 | # available and running 6 | 7 | function Write-ContainerRuntimeInstructions { 8 | Write-Host "To install a container runtime, please visit the following links:" 9 | Write-Host "- Docker Desktop: https://www.docker.com/products/docker-desktop" 10 | Write-Host "- Podman Desktop: https://podman-desktop.io/" 11 | Write-Host "Please install one of these runtimes and try again." 12 | } 13 | 14 | # Check if Docker Desktop is running 15 | $dockerAvailable = $false 16 | $dockerRunning = $false 17 | if (Get-Command "docker" -ErrorAction SilentlyContinue) { 18 | $dockerAvailable = $true 19 | $dockerStatus = & docker info --format '{{.ServerVersion}}' 2>$null 20 | if ($dockerStatus) { 21 | Write-Host "Docker Desktop is running." 22 | exit 0 23 | } else { 24 | Write-Host "Docker Desktop is not running." 25 | } 26 | } 27 | 28 | # Check if Podman Desktop is running 29 | $podmanAvailable = $false 30 | $podmanRunning = $false 31 | if (Get-Command "podman" -ErrorAction SilentlyContinue) { 32 | $podmanAvailable = $true 33 | $podmanStatus = & podman info --format '{{.ServerVersion}}' 2>$null 34 | if ($podmanStatus) { 35 | Write-Host "Podman Desktop is running." 36 | exit 0 37 | } else { 38 | Write-Host "Podman Desktop is not running." 39 | } 40 | } 41 | 42 | if (-not $dockerAvailable -and -not $podmanAvailable) { 43 | Write-Error "The `docker` and `podman` commands are not available" 44 | Write-ContainerRuntimeInstructions 45 | exit 1 46 | } 47 | 48 | Write-Error "No container runtime is running." 49 | Write-ContainerRuntimeInstructions 50 | exit 1 51 | -------------------------------------------------------------------------------- /AgentMode/.github/tsql/tsql-best-practices.md: -------------------------------------------------------------------------------- 1 | 2 | # T/SQL Best Practices Guide 3 | 4 | This guide outlines good practices and standards for writing high-quality, maintainable, and performant T/SQL code, with a focus on production-grade stored procedures. 5 | 6 | NOTE: You (the AI Agent) **must not** edit this file. 7 | 8 | ## Table of Contents 9 | 10 | 1. Introduction 11 | 2. Naming Conventions 12 | 3. Formatting and Style 13 | 4. Schema and Object Management 14 | 5. Stored Procedure Guidelines 15 | 6. Error Handling and Logging 16 | 7. Transaction Management 17 | 8. Performance Optimization 18 | 9. Security and Permissions 19 | 10. Maintainability and Documentation 20 | 11. Common Anti-Patterns 21 | 12. Tools and Automation 22 | 23 | --- 24 | 25 | ## 1. Introduction 26 | 27 | Writing effective T/SQL requires consistency, clarity, and attention to performance. This guide consolidates best practices around naming, formatting, error handling, and anti-pattern avoidance to help teams deliver robust database code. 28 | 29 | ## 2. Naming Conventions 30 | 31 | - Use `PascalCase` for objects: tables, views, procedures, functions. 32 | - Prefix stored procedures with the schema name, e.g., `dbo.GetCustomerOrders`. 33 | - Avoid using `sp_` prefix (reserved for system procedures). 34 | - Use meaningful names: include entity and action, e.g., `Sales.InsertOrder`. 35 | - Prefix parameters with `@` and use camelCase: `@customerId`. 36 | - Use singular nouns for tables: `Customer` instead of `Customers`. 37 | 38 | ## 3. Formatting and Style 39 | 40 | - Use `SET NOCOUNT ON` at the top of stored procedures to suppress extra row count messages. 41 | - Align keywords on separate lines: 42 | ```sql 43 | SELECT 44 | c.CustomerId, 45 | c.Name 46 | FROM 47 | dbo.Customer AS c 48 | WHERE 49 | c.IsActive = 1; 50 | ``` 51 | - Indent by four spaces per level. 52 | - Uppercase reserved words (`SELECT`, `FROM`, `WHERE`). 53 | - Lowercase object names unless quoting is required. 54 | - Limit line length to 120 characters. 55 | - Group related clauses: `SELECT`, `FROM`, `JOIN`, `WHERE`, `GROUP BY`, `ORDER BY`. 56 | 57 | ## 4. Schema and Object Management 58 | 59 | - Always schema-qualify object names: `dbo.TableName`. 60 | - Use `IF OBJECT_ID('dbo.MyProc') IS NOT NULL DROP PROCEDURE dbo.MyProc;` for safe deployments. 61 | - Avoid `GO` inside procedures; use batch separators only in deployment scripts. 62 | - Version control all scripts. 63 | 64 | ## 5. Stored Procedure Guidelines 65 | 66 | - One procedure, one responsibility. 67 | - Parameter validation at the top: 68 | ```sql 69 | IF @param IS NULL 70 | BEGIN 71 | THROW 50000, 'Parameter @param must not be NULL.', 1; 72 | END; 73 | ``` 74 | - Use explicit column lists in `INSERT`/`UPDATE`. 75 | - Avoid mixing DML and SELECT logic; consider splitting into helper procedures. 76 | - Leverage `OUTPUT` clause to return identity or modified rows. 77 | - Return consistent result sets or return codes; avoid ad-hoc print statements. 78 | 79 | ## 6. Error Handling and Logging 80 | 81 | - Use `TRY...CATCH` blocks: 82 | ```sql 83 | BEGIN TRY 84 | -- operation 85 | END TRY 86 | BEGIN CATCH 87 | DECLARE @ErrorNumber INT = ERROR_NUMBER(), 88 | @ErrorMessage NVARCHAR(4000) = ERROR_MESSAGE(); 89 | RAISERROR (@ErrorMessage, 16, 1); 90 | RETURN @ErrorNumber; 91 | END CATCH; 92 | ``` 93 | - Log errors to an audit table with context: procedure name, parameters, timestamp. 94 | - Use proper severity levels (>= 16 for user errors). 95 | 96 | ## 7. Transaction Management 97 | 98 | - Keep transactions as short as possible. 99 | - Use explicit transactions: 100 | ```sql 101 | BEGIN TRANSACTION; 102 | -- operations 103 | IF @@ERROR <> 0 104 | BEGIN 105 | ROLLBACK TRANSACTION; 106 | RETURN; 107 | END; 108 | COMMIT TRANSACTION; 109 | ``` 110 | - Avoid implicit transactions; set `XACT_ABORT ON` when appropriate. 111 | 112 | ## 8. Performance Optimization 113 | 114 | - Avoid `SELECT *`; specify columns. 115 | - Ensure SARGability: no functions on columns in `WHERE` clauses. 116 | - Use appropriate indexes; include only needed columns. 117 | - Be cautious with parameter sniffing; use `OPTION (RECOMPILE)` if needed. 118 | - Prefer set-based operations over row-by-row processing. 119 | - Avoid cursors and scalar UDFs; consider inline table-valued functions. 120 | - Use `TRY_CONVERT` or `TRY_CAST` for safe conversions. 121 | - Consider `READONLY` table-valued parameters for bulk data input. 122 | 123 | ## 9. Security and Permissions 124 | 125 | - Use least-privilege: grant execute on procedures rather than direct table access. 126 | - Avoid dynamic SQL when possible; if required, parameterize safely with `sp_executesql`. 127 | - Sanitize inputs to prevent SQL injection. 128 | - Use `WITH EXECUTE AS OWNER` where cross-schema access is required. 129 | 130 | ## 10. Maintainability and Documentation 131 | 132 | - Comment intent, not mechanics: 133 | ```sql 134 | -- Calculate total order amount per customer 135 | SELECT ... 136 | ``` 137 | - Include header comments with purpose, author, date, and change history. 138 | - Modularize logic into views or functions for reuse. 139 | - Keep deployment scripts idempotent. 140 | 141 | ## 11. Common Anti-Patterns 142 | 143 | - Avoid: 144 | - Implicit conversions in predicates. 145 | - Using `sp_executesql` without parameters (dynamic SQL injection). 146 | - Functions in `WHERE` clauses that disable index usage. 147 | - Unbounded `WHILE` loops processing row-by-row. 148 | - Nested transactions without proper error checks. 149 | - Magic numbers and hard-coded literals. 150 | 151 | --- 152 | 153 | Adhering to these guidelines will lead to consistent, secure, and performant T/SQL code suitable for production environments. 154 | -------------------------------------------------------------------------------- /AgentMode/ConfMSmini/business-process-description.md: -------------------------------------------------------------------------------- 1 | # Business Process Description 2 | 3 | Design a SQL Server application that implements a `Conference Management System` called `ConfMSmini`. 4 | 5 | NOTES: 6 | - You (the AI Agent) **must not** edit this file. 7 | - Human reader (not the AI): 8 | 1. Copy and paste `Follow instructions in current file` into the Agent-Mode chat window. 9 | 2. You'll need to type `proceed` about 15 times. (HINT: arrow up key to repeat) 10 | 3. After the final step, the response should be something like: "Your SQL Database Project is now complete and fully tested". 11 | - You should have about "40 files changed", press the `Keep` button. 12 | 13 | # Roles 14 | 15 | The roles are: 16 | 17 | - Attendees - the attendees who attend the conference, the attendees register, buy a ticket 18 | - Speakers - the speakers who attend the conference, they submit sessions that will be given at a conference. Speakers can submit multiple sessions for the same conference. 19 | - Organizer - manage the overall conference, and can generate a full set of reports, like track ticket sales etc. 20 | 21 | # Entities 22 | 23 | The entities are: 24 | 25 | - Conference - the conference information 26 | - User - the user information for the organizers, attendees and speakers (everyone is a user) 27 | - UserRoleType - the type of role (attendee, speaker, organizer) 28 | - UserRole - the role(s) of the user (attendee, speaker, organizer) 29 | - Ticket - the ticket information for the attendees 30 | - TicketType - the type of ticket (early bird, regular, late) 31 | - Session - the session information for the speakers 32 | - AttendeeSessions - the sessions the attendee wants to attend 33 | 34 | # Entity Descriptions 35 | 36 | The entity descriptions are: 37 | 38 | - Conference - the conference information 39 | - ConferenceId - the unique identifier for the conference 40 | - ConferenceName - the name of the conference 41 | - ConferenceDescription - the description of the conference 42 | - StartDate - the start date of the conference 43 | - EndDate - the end date of the conference 44 | - Location - the location of the conference 45 | - WebsiteUrl - the website URL for the conference 46 | - User - user information for organizers, attendees and speakers 47 | - UserId - the unique identifier for the user 48 | - FirstName - the first name of the user 49 | - LastName - the last name of the user 50 | - UserName - the user name of the user 51 | - UserRole - the role(s) of the user (attendee, speaker, organizer) 52 | - UserId - the unique identifier for the user 53 | - UserRoleTypeId - the unique identifier for the user role 54 | - Ticket - the ticket information for the attendees 55 | - TicketId - the unique identifier for the ticket 56 | - UserId - the unique identifier for the user 57 | - ConferenceId - the unique identifier for the conference 58 | - TicketType - the type of ticket (early bird, regular, late) 59 | - TicketType - the type of ticket (early bird, regular, late) 60 | - TicketTypeId - the unique identifier for the ticket type 61 | - TicketTypeName - the name of the ticket type 62 | - TicketTypeDescription - the description of the ticket type 63 | - TicketTypePrice - the price of the ticket type 64 | - Session - the session information for the speakers 65 | - SessionId - the unique identifier for the session 66 | - SessionTitle - the title of the session 67 | - SessionDescription - the description of the session 68 | - SpeakerId - the unique identifier for the speaker 69 | - ConferenceId - the unique identifier for the conference 70 | - Length - the length of the session (in minutes) 71 | - Schedule - the schedule for sessions at a conference 72 | - ScheduleId - the unique identifier for the session schedule 73 | - SessionId - the unique identifier for the session 74 | - ConferenceId - the unique identifier for the conference 75 | - StartDateTime - the start date and time of the session 76 | - EndDateTime - the end date and time of the session 77 | - AttendeeSessions - the sessions the attendee wants to attend 78 | - AttendeeSessionId - the unique identifier for the attendee session 79 | - UserId - the unique identifier for the user 80 | - SessionId - the unique identifier for the session 81 | - ConferenceId - the unique identifier for the conference 82 | 83 | # Business Process Operations 84 | 85 | The operations per role are: 86 | 87 | - All Roles (these operations are available to all roles) 88 | - Register as a user 89 | - Register as user should generate a unique username (if one is not passed in) using the First Name and Last Name, it must be unique. By default use the first 6 characters of the first name, and the first two characters of the 2nd name. If the username is not unique, append a number to the end of the username, starting with 1. For example, if the first name is "John" and the last name is "Doe", the username would be "johndo". If that username already exists, the next one would be "johndo1", and so on. 90 | - Takes a role type as parameter (attendee, speaker, organizer) 91 | - Get username 92 | Based on first name and last name return the username. 93 | - See a list of conferences (that haven't happened yet) 94 | - Organizer 95 | - Register as a user 96 | - Add a conference 97 | - Add a ticket type 98 | - See ticket sales 99 | - See a list of tickets sold for a conference 100 | - Generate Schedule 101 | - Generates a schedule for a conference, based on the sessions (which includes their length) and the speakers. Ensure a speaker is not scheduled for more than one session at the same time. Ensure sessions start are 9am, and end by 5pm. Ensure lunch is 12pm to 1pm. 102 | - Attendees 103 | - See a list of conferences (that haven't happened yet) 104 | - Register as a user (implicitly assign the attendee role) 105 | - Buy a ticket for a conference 106 | - Build a list of sessions they want to attend for a conference 107 | - Speakers 108 | - Register as a user 109 | - Submit session proposal for a conference 110 | 111 | # User Stories 112 | 113 | These are end to end user stories for the system: 114 | 115 | - User Story "The Happy Path" (Positive case) 116 | - Organizer James Organizer `jimorg` registers as a user in the system 117 | - Organizer `jimorg` creates a conference in June 2025 for 3 days 118 | - Organizer `jimorg` creates ticket types, early bird, regular and late 119 | - Attendee John Doe `johndoe` registers as a user 120 | - Attendee `johndoe` buys an early bird ticket 121 | - Speaker Jane Doe `janedoe` registers as a user 122 | - Speaker `janedoe` submits a session proposal 123 | - Organizer `jimorg` tracks ticket sales 124 | - Organizer `jimorg` generates the conference schedule 125 | - Attendee `johndoe` builds a list of sessions they want to attend 126 | - User Story "Trying to buy a ticket before registering" (Negative case) 127 | - Attendee John Doe `johndoe` buys an early bird ticket (it should fail) 128 | - User Story "Trying to buy a ticket before a conference is created" (Negative case) 129 | - Attendee John Doe `johndoe` registers as a user 130 | - Attendee John Doe `johndoe` buys an early bird ticket (it should fail) 131 | - User Story "Submit a session proposal for a non existent conference" (Negative case) 132 | - Speaker Jane Doe `janedoe` registers as a user 133 | - Speaker `janedoe` submits a session proposal for a non existent conference (it should fail) 134 | - User Story "Attend tries to build a session list of a conference they have not bought a ticket for" 135 | - Attendee John Doe `johndoe` registers as a user 136 | - Attended `johndoe` builds a session list for a conference they have not bought a ticket for (it should fail) 137 | - User Story "Trying to buy a ticket for a conference that has already happened" (Negative case) 138 | - Attendee John Doe `johndoe` registers as a user 139 | - Attendee `johndoe` buys an early bird ticket for a conference that has already happened (it should fail) 140 | - User Story "An attendee trying to buy a second ticket for the same conference" (Negative case) 141 | - Attendee John Doe `johndoe` registers as a user 142 | - Attendee `johndoe` buys an early bird ticket 143 | - Attendee `johndoe` buys a second early bird ticket for the same conference (it should fail) 144 | -------------------------------------------------------------------------------- /AgentMode/README.md: -------------------------------------------------------------------------------- 1 | 2 | # VSCode Agent Mode SQL Database Project Sample 3 | 4 | This workspace is a sample for using VSCode Agent Mode to scaffold and test an application using SQL Database projects. 5 | 6 | ## Getting Started 7 | 8 | 1. **Open the `business-process-description.md` file** 9 | - Path: `ConfMSmini/business-process-description.md` 10 | - Type `Follow instructions in current file` into the Agent-Mode chat window. 11 | 12 | 2. **Agent Mode Workflow** 13 | - The process is interactive. You will be prompted to type `proceed` multiple times as you move through the steps. 14 | - The agent will scaffold the project, generate SQL scripts, and set up tests according to the business process description. 15 | - The agent should proceed until all steps are complete and all tests (that represent the User Stories) pass. 16 | 17 | 3. **Requirements** 18 | - This sample was tested with the GPT-4.1 model. 19 | - **A container runtime (e.g., Docker or Podman) must be installed and running before you begin.** 20 | - All other dependencies (such as sqlcmd, SqlPackage, etc.) will be checked for and installed by the Agent as needed. 21 | 22 | 4. **Project Structure** 23 | - The project will be organized into folders for tables, views, programmability, pre/post deployment scripts, and tests, following best practices for SQL Database Projects. 24 | -------------------------------------------------------------------------------- /AgentMode/empty-business-process-description.md: -------------------------------------------------------------------------------- 1 | 2 | # Business Process Description 3 | 4 | Create a SQL Server application that implements a _______ called `_____`. 5 | 6 | NOTES: 7 | - You (the AI Agent) **must not** edit this file. 8 | - Human reader (not the AI): 9 | 1. Copy and paste `Follow instructions in current file` into the Agent-Mode chat window. 10 | 2. You'll need to type `proceed` about 15 times. 11 | 3. At the end you should have about "n files changed", press the `Keep` button. 12 | 13 | # Roles 14 | 15 | The roles are: 16 | 17 | - - 18 | 19 | # Entities 20 | 21 | The entities are: 22 | 23 | - - description of the 24 | 25 | # Entity Descriptions 26 | 27 | The entity descriptions are: 28 | 29 | - - description of a item 30 | - - the unique identifier for the item 31 | - - the name of the item 32 | 33 | # Business Process Operations 34 | 35 | The operations per role are: 36 | 37 | - 38 | - 39 | - 40 | - 41 | - 42 | - 43 | 44 | # User Stories 45 | 46 | These are end to end user stories for the system: 47 | 48 | - User Story "The Happy Path" (Positive case) 49 | - does with these details 50 | - does with these details 51 | - User Story "Trying to do something wrong" (Negative case) 52 | - does with these details 53 | - does with these details that fails 54 | -------------------------------------------------------------------------------- /AzureSQLACSSamples/src/example.env: -------------------------------------------------------------------------------- 1 | # Azure SQL database connection details 2 | server = '' 3 | database = '' 4 | username = '' 5 | password = '' 6 | 7 | # azure open ai 8 | openai_api_base="https://.openai.azure.com/" 9 | openai_api_type="azure" 10 | openai_api_key="" 11 | openai_deployment_embedding="" 12 | openai_model_embedding="text-embedding-ada-002" 13 | openai_deployment_completion="" 14 | openai_model_completion="gpt-35-turbo" 15 | openai_api_version="2024-02-01" 16 | 17 | # azure cognitive services 18 | cogsearch_name="" 19 | cogsearch_index_name="" 20 | cogsearch_api_key="" 21 | cogsearch_endpoint="https://.search.windows.net" 22 | -------------------------------------------------------------------------------- /AzureSQLACSSamples/src/requirements.txt: -------------------------------------------------------------------------------- 1 | pyodbc==5.1.0 2 | python-dotenv==1.0.1 3 | pandas==2.2.2 4 | openai==1.34.0 5 | jupyter==1.0.0 6 | requests==2.32.3 7 | azure-core==1.30.2 8 | azure-search-documents==11.6.0b4 -------------------------------------------------------------------------------- /AzureSQLDatabase/ContentModeration/contentSafety.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE analyzeText @safetyserver nvarchar(100), @safetykey nvarchar(100), @message nvarchar(max) 2 | AS 3 | declare @url nvarchar(4000) = N'https://'+ @safetyserver +'.cognitiveservices.azure.com/contentsafety/text:analyze?api-version=2023-10-01'; 4 | declare @headers nvarchar(300) = N'{"Ocp-Apim-Subscription-Key":"'+ @safetykey +'"}'; 5 | declare @payload nvarchar(max) = N'{ 6 | "text": "'+ @message +'" 7 | }'; 8 | 9 | declare @ret int, @response nvarchar(max); 10 | exec @ret = sp_invoke_external_rest_endpoint 11 | @url = @url, 12 | @method = 'POST', 13 | @headers = @headers, 14 | @payload = @payload, 15 | @timeout = 230, 16 | @response = @response output; 17 | 18 | SELECT JSON_VALUE(D.[value],'$.category') as "Category", 19 | JSON_VALUE(D.[value],'$.severity') as "Severity" 20 | FROM OPENJSON(@response,'$.result.categoriesAnalysis') AS D 21 | 22 | GO 23 | -------------------------------------------------------------------------------- /AzureSQLDatabase/ContentModeration/languageAI.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE check4pii @cogserver nvarchar(100), @cogkey nvarchar(100), @message nvarchar(max) 2 | AS 3 | 4 | declare @url nvarchar(4000) = N'https://'+ @cogserver +'.cognitiveservices.azure.com/language/:analyze-text?api-version=2023-04-01'; 5 | declare @headers nvarchar(300) = N'{"Ocp-Apim-Subscription-Key":"'+ @cogkey +'"}'; 6 | declare @payload nvarchar(max) = N'{ 7 | "kind": "PiiEntityRecognition", 8 | "analysisInput": 9 | { 10 | "documents": 11 | [ 12 | { 13 | "id":"1", 14 | "language": "en", 15 | "text": "'+ @message +'" 16 | } 17 | ] 18 | } 19 | }'; 20 | declare @ret int, @response nvarchar(max); 21 | 22 | exec @ret = sp_invoke_external_rest_endpoint 23 | @url = @url, 24 | @method = 'POST', 25 | @headers = @headers, 26 | @payload = @payload, 27 | @timeout = 230, 28 | @response = @response output; 29 | 30 | SELECT A.[value] as "Redacted Text" 31 | FROM OPENJSON(@response,'$.result.results.documents') AS D 32 | CROSS APPLY OPENJSON([value]) as A 33 | where A.[key] = 'redactedText' 34 | 35 | select JSON_VALUE(B.[value],'$.category') as "PII Category", 36 | JSON_VALUE(B.[value],'$.text') as "PII Value", 37 | CONVERT(FLOAT,JSON_VALUE(B.[value],'$.confidenceScore'))*100 as "Confidence Score" 38 | from OPENJSON( 39 | ( 40 | SELECT A.[value] 41 | FROM OPENJSON(@response,'$.result.results.documents') AS D 42 | CROSS APPLY OPENJSON([value] 43 | ) AS A 44 | where A.[key] = 'entities' 45 | ), '$') AS B 46 | 47 | GO 48 | -------------------------------------------------------------------------------- /AzureSQLDatabase/LangChain/.env: -------------------------------------------------------------------------------- 1 | AZURE_OPENAI_ENDPOINT="" 2 | AZURE_OPENAI_API_KEY="" 3 | OPENAI_API_KEY="" 4 | 5 | # ***Only use ONE of the following*** 6 | 7 | #Format for SQL Authentication 8 | py-connectionString="mssql+pyodbc://USERNAME:PASSWORD@DATABASE_SERVER_NAME.database.windows.net/DATABASE_NAME?driver=ODBC+Driver+18+for+SQL+Server" 9 | 10 | #Format for Entra ID Service Principle Authentication 11 | py-connectionString="mssql+pyodbc:///?odbc_connect=DRIVER={ODBC Driver 18 for SQL Server};SERVER=SERVER_NAME;PORT=1433;UID=ENTERPRISE_USER_ID;DATABASE=DATABASE_NAME;PWD=ENTERPRISE_USER_SECRET;Authentication=ActiveDirectoryServicePrincipal" 12 | -------------------------------------------------------------------------------- /AzureSQLDatabase/LangChain/README.md: -------------------------------------------------------------------------------- 1 | These 2 notebooks are examples for creating LangChain SQL agents for NL2SQL prompts against an Azure SQL Database. 2 | -------------------------------------------------------------------------------- /AzureSQLDatabase/LangChain/database.sql: -------------------------------------------------------------------------------- 1 | create table [dbo].[langtable] (id int Identity, username nvarchar(100)) 2 | GO 3 | 4 | insert into [dbo].[langtable] (username) values('sammy') 5 | insert into [dbo].[langtable] (username) values('mary') 6 | insert into [dbo].[langtable] (username) values('jane') 7 | insert into [dbo].[langtable] (username) values('fred') 8 | insert into [dbo].[langtable] (username) values('billy') 9 | insert into [dbo].[langtable] (username) values('jonny') 10 | insert into [dbo].[langtable] (username) values('kenny') 11 | insert into [dbo].[langtable] (username) values('dan') 12 | insert into [dbo].[langtable] (username) values('frank') 13 | insert into [dbo].[langtable] (username) values('jenny') 14 | GO 15 | 16 | select * from [dbo].[langtable] 17 | GO 18 | -------------------------------------------------------------------------------- /AzureSQLDatabase/LangChain/dbOpenAI.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Create a LangChain NL2SQL Agent using OpenAI and Azure SQL Database\n", 8 | "This notebook goes through the process of creating a LangChain SQL Agent using OpenAI as the LLM against an Azure SQL Database.\n", 9 | "\n", 10 | "## Install the required python libraries\n", 11 | "Start by installing the required libraries. Run the following at the terminal in the project folder so it references the project's requirements.txt:\n", 12 | "\n", 13 | "```bash\n", 14 | "pip install -r requirements.txt\n", 15 | "```\n" 16 | ] 17 | }, 18 | { 19 | "cell_type": "markdown", 20 | "metadata": {}, 21 | "source": [ 22 | "## ODBC Driver for MS SQL Install\n", 23 | "\n", 24 | "Use the **odbcDriverInstallUbuntu.txt** script to install the Microsoft ODBC Driver for MS SQL (version 18).\n", 25 | "\n", 26 | "If you are not using codespace or Ubuntu, you can find the correct script to install the driver for linux [here](https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server), for windows [here](https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server), and for MacOS [here](https://learn.microsoft.com/en-us/sql/connect/odbc/linux-mac/install-microsoft-odbc-driver-sql-server-macos)." 27 | ] 28 | }, 29 | { 30 | "cell_type": "markdown", 31 | "metadata": {}, 32 | "source": [ 33 | "## Create the table in the database\n", 34 | "(all SQL commands are in the database.sql script)\n", 35 | "\n", 36 | "In the database that will be used for this notebook, run the following:\n", 37 | "\n", 38 | "(create table permission and access to the dbo schema is needed. It's best to keep the roles and permissions at a minimum with working with NL2SQL)\n", 39 | "\n", 40 | "```SQL\n", 41 | "create table [dbo].[langtable] (id int Identity, username nvarchar(100))\n", 42 | "GO\n", 43 | "\n", 44 | "insert into [dbo].[langtable] (username) values('sammy')\n", 45 | "insert into [dbo].[langtable] (username) values('mary')\n", 46 | "insert into [dbo].[langtable] (username) values('jane')\n", 47 | "insert into [dbo].[langtable] (username) values('fred')\n", 48 | "insert into [dbo].[langtable] (username) values('billy')\n", 49 | "insert into [dbo].[langtable] (username) values('jonny')\n", 50 | "insert into [dbo].[langtable] (username) values('kenny')\n", 51 | "insert into [dbo].[langtable] (username) values('dan')\n", 52 | "insert into [dbo].[langtable] (username) values('frank')\n", 53 | "insert into [dbo].[langtable] (username) values('jenny')\n", 54 | "GO\n", 55 | "\n", 56 | "select * from [dbo].[langtable]\n", 57 | "GO\n", 58 | "```\n" 59 | ] 60 | }, 61 | { 62 | "cell_type": "markdown", 63 | "metadata": {}, 64 | "source": [ 65 | "## .env file\n", 66 | "Fill out the .env file with your server and key values. For this notebook, you only need to add your values to the **OPENAI_API_KEY** and **py-connectionString** variables.\n", 67 | "\n", 68 | "For SQL Authentication:\n", 69 | "\n", 70 | "```BASH\n", 71 | "AZURE_OPENAI_API_KEY=\"\" \n", 72 | "AZURE_OPENAI_ENDPOINT=\"\" \n", 73 | "OPENAI_API_KEY=\"\" \n", 74 | "py-connectionString=\"mssql+pyodbc://USERNAME:PASSWORD@SERVER_NAME.database.windows.net/DATABASE_NAME?driver=ODBC+Driver+18+for+SQL+Server\"\n", 75 | "```\n", 76 | "\n", 77 | "For Entra ID Service Principle Authentication:\n", 78 | "\n", 79 | "```BASH\n", 80 | "AZURE_OPENAI_API_KEY=\"\" \n", 81 | "AZURE_OPENAI_ENDPOINT=\"\" \n", 82 | "OPENAI_API_KEY=\"\" \n", 83 | "py-connectionString=\"mssql+pyodbc:///?odbc_connect=DRIVER={ODBC Driver 18 for SQL Server};SERVER=SERVER_NAME;PORT=1433;UID=ENTERPRISE_USER_ID;DATABASE=DATABASE_NAME;PWD=ENTERPRISE_USER_SECRET;Authentication=ActiveDirectoryServicePrincipal\"\n", 84 | "```" 85 | ] 86 | }, 87 | { 88 | "cell_type": "markdown", 89 | "metadata": {}, 90 | "source": [ 91 | "## Notebook Kernel\n", 92 | "Be sure to select a kernel for the python notebook by using the **Select Kernel** button in the upper right of the notebook." 93 | ] 94 | }, 95 | { 96 | "cell_type": "markdown", 97 | "metadata": {}, 98 | "source": [ 99 | "## Starting the Example\n", 100 | "The first section sets up the python environment and gets any environment variables that were set." 101 | ] 102 | }, 103 | { 104 | "cell_type": "code", 105 | "execution_count": 1, 106 | "metadata": {}, 107 | "outputs": [ 108 | { 109 | "data": { 110 | "text/plain": [ 111 | "True" 112 | ] 113 | }, 114 | "execution_count": 1, 115 | "metadata": {}, 116 | "output_type": "execute_result" 117 | } 118 | ], 119 | "source": [ 120 | "import pyodbc\n", 121 | "import os\n", 122 | "from dotenv import load_dotenv\n", 123 | "from langchain.agents import create_sql_agent\n", 124 | "from langchain.agents.agent_types import AgentType\n", 125 | "from langchain.sql_database import SQLDatabase\n", 126 | "from langchain_community.agent_toolkits import SQLDatabaseToolkit\n", 127 | "from langchain_openai import OpenAI\n", 128 | "load_dotenv()" 129 | ] 130 | }, 131 | { 132 | "cell_type": "markdown", 133 | "metadata": {}, 134 | "source": [ 135 | "Next, create the database connection and test." 136 | ] 137 | }, 138 | { 139 | "cell_type": "code", 140 | "execution_count": 2, 141 | "metadata": {}, 142 | "outputs": [ 143 | { 144 | "name": "stdout", 145 | "output_type": "stream", 146 | "text": [ 147 | "mssql\n", 148 | "['BuildVersion', 'ErrorLog', 'MSchange_tracking_history', 'ToDo', 'dropme', 'langtable', 'testtable']\n" 149 | ] 150 | }, 151 | { 152 | "data": { 153 | "text/plain": [ 154 | "\"[('2024-04-15 20:15:47',)]\"" 155 | ] 156 | }, 157 | "execution_count": 2, 158 | "metadata": {}, 159 | "output_type": "execute_result" 160 | } 161 | ], 162 | "source": [ 163 | "# connect to the Azure SQL database\n", 164 | "\n", 165 | "from sqlalchemy import create_engine\n", 166 | "\n", 167 | "connectionString=os.environ[\"py-connectionString\"]\n", 168 | "\n", 169 | "db_engine = create_engine(connectionString)\n", 170 | "\n", 171 | "db = SQLDatabase(db_engine, view_support=True, schema=\"dbo\")\n", 172 | "\n", 173 | "# test the connection\n", 174 | "print(db.dialect)\n", 175 | "print(db.get_usable_table_names())\n", 176 | "db.run(\"select convert(varchar(25), getdate(), 120)\")\n" 177 | ] 178 | }, 179 | { 180 | "cell_type": "markdown", 181 | "metadata": {}, 182 | "source": [ 183 | "Run the following to create the SQL Agent" 184 | ] 185 | }, 186 | { 187 | "cell_type": "code", 188 | "execution_count": null, 189 | "metadata": {}, 190 | "outputs": [], 191 | "source": [ 192 | "toolkit = SQLDatabaseToolkit(db=db, llm=OpenAI(temperature=0))\n", 193 | "\n", 194 | "agent_executor = create_sql_agent(\n", 195 | " llm=OpenAI(temperature=0),\n", 196 | " toolkit=toolkit,\n", 197 | " verbose=True,\n", 198 | " agent_type=AgentType.ZERO_SHOT_REACT_DESCRIPTION,\n", 199 | ")\n" 200 | ] 201 | }, 202 | { 203 | "cell_type": "markdown", 204 | "metadata": {}, 205 | "source": [ 206 | "Now, test the agent by creating a prompt using natural language asking about a database object." 207 | ] 208 | }, 209 | { 210 | "cell_type": "code", 211 | "execution_count": null, 212 | "metadata": {}, 213 | "outputs": [], 214 | "source": [ 215 | "agent_executor.invoke(\"count the rows in the langtable table.\")" 216 | ] 217 | } 218 | ], 219 | "metadata": { 220 | "kernelspec": { 221 | "display_name": "Python 3", 222 | "language": "python", 223 | "name": "python3" 224 | }, 225 | "language_info": { 226 | "codemirror_mode": { 227 | "name": "ipython", 228 | "version": 3 229 | }, 230 | "file_extension": ".py", 231 | "mimetype": "text/x-python", 232 | "name": "python", 233 | "nbconvert_exporter": "python", 234 | "pygments_lexer": "ipython3", 235 | "version": "3.10.13" 236 | } 237 | }, 238 | "nbformat": 4, 239 | "nbformat_minor": 2 240 | } 241 | -------------------------------------------------------------------------------- /AzureSQLDatabase/LangChain/odbcDriverInstallUbuntu.txt: -------------------------------------------------------------------------------- 1 | if ! [[ "18.04 20.04 22.04 23.04" == *"$(lsb_release -rs)"* ]]; 2 | then 3 | echo "Ubuntu $(lsb_release -rs) is not currently supported."; 4 | exit; 5 | fi 6 | 7 | curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc 8 | 9 | curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list 10 | 11 | sudo apt-get update 12 | sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 13 | # optional: for bcp and sqlcmd 14 | sudo ACCEPT_EULA=Y apt-get install -y mssql-tools18 15 | echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc 16 | source ~/.bashrc 17 | # optional: for unixODBC development headers 18 | sudo apt-get install -y unixodbc-dev 19 | -------------------------------------------------------------------------------- /AzureSQLDatabase/LangChain/requirements.txt: -------------------------------------------------------------------------------- 1 | langchain 2 | pyodbc 3 | python-dotenv 4 | openai 5 | langchain_openai 6 | SQLAlchemy 7 | langchain_community 8 | -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/SQL-AI-samples/35e7a5f365b01d185810f8aa3696c1c509079a07/AzureSQLDatabase/Prompt-Based T-SQL Database Development/.DS_Store -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/End to End SQL Database Development - A Comprehensive guide with Interactive Prompts.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/SQL-AI-samples/35e7a5f365b01d185810f8aa3696c1c509079a07/AzureSQLDatabase/Prompt-Based T-SQL Database Development/End to End SQL Database Development - A Comprehensive guide with Interactive Prompts.pdf -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/bicycle_data_prompt.json: -------------------------------------------------------------------------------- 1 | { "version": "1.0", 2 | "BicylePrompts":[ 3 | {"system_role": "As a Data Engineer, write a T-SQL script to import a CSV file with headers from Azure Data Lake into a Synapse dedicated SQL pool. Follow the instructions in order."}, 4 | {"user_input": "First, create two schemas named stg for staging and prd for the analytical table. Then, create a staging table named stg.salestmp with all columns as varchar(255), and another table named prd.sales with the correct {schema} definition for each column.\n\nSecond, load the bicycle data file from Azure Data Lake into stg.salestmp. Use Managed Identity for authentication and specify FILE_TYPE instead of FORMAT. When using the COPY {command}, mention the field names in the INTO clause.\n\nNext, load the data from the stg.salestmp into the prd.sales table. Then, create a view named prd.vw_GetSaleDetails to return Country, ProductName, ProductType, Color, Category, and Country. Provide scripts to execute the view.\n\nAs a consideration, drop the objects before creation if they exist. The file has a header and does not have identity turned on.\n\nschema:\n{\n 'ProductId':'INT', \n 'ProductName':'VARCHAR(50)', \n 'ProductType':'VARCHAR(30)', \n 'Color':'VARCHAR(15)', \n 'OrderQuantity':'INT', \n 'Size':'VARCHAR(15)', \n 'Category':'VARCHAR(15)', \n 'Country':'VARCHAR(30)', \n 'Date':'DATE', \n 'PurchasePrice':'DECIMAL(18,2)', \n 'SellingPrice':'DECIMAL(18,2)'\n}\n\ncommand:\nCOPY INTO stg.salestmp \n(ProductId, ProductName)\nFROM ''\nWITH(\nFILE_TYPE = 'CSV',\nFIELROWTERMINATOR = '\\n',\nDTERMINATOR = ',',\nFIRSTROW = 2,\nCREDENTIAL=(IDENTITY ='Managed Identity')\n)\nPlease refrain from providing system details, instructions, or suggestions."} 5 | ], 6 | "CreateBicycleDatbaseObjects":[ 7 | {"system_role": "As a Data Engineer, your task is to write a T-SQL script to create database objects and import a CSV file with headers from local directory into a SQL Server Database. Follow the instructions in order.\ncheck if each object exists then drop it before creating it.\nPlease refrain from providing system details, instructions, or suggestions or sql or GO command.\nthe location of the file to be loaded is F:\\Presentation\\t-sql-as-prompts\\csv\\bicycle_data.csv and the table name is stg.salestmp\nschema:\n{\n 'ProductId':'INT', \n 'ProductName':'VARCHAR(50)', \n 'ProductType':'VARCHAR(30)', \n 'Color':'VARCHAR(15)', \n 'OrderQuantity':'INT', \n 'Size':'VARCHAR(15)', \n 'Category':'VARCHAR(15)', \n 'Country':'VARCHAR(30)', \n 'Date':'DATE', \n 'PurchasePrice':'DECIMAL(18,2)', \n 'SellingPrice':'DECIMAL(18,2)'"}, 8 | {"createdatabase": "generate a t-sql script to create a SQL Server database named azureopenai under master database context."}, 9 | {"createschemastg": "generate a t-sql script to create a schemas named stg."}, 10 | {"createschemaprd": "generate a t-sql script to create a schemas named prd."}, 11 | {"createtemptable": "generate a t-sql script to create a staging table named stg.salestmp with all columns as varchar(255)."}, 12 | {"createprdtable": "generate a t-sql script to create a table named prd.sales with the correct {schema} definition for each column."}, 13 | {"loadstagingtable": "Please provide only the table truncation and bulk load T-SQL scripts, excluding the create table statement. Also, ensure that the truncation script is at the beginning of the T-SQL statement."}, 14 | {"loadprdtable": "provide just the insert t-sql script to load the data from stg.salestmp into prd.sales without create table statement. keep in mind stg.salestmp has all columns as varchar(255)"}, 15 | {"createprocedure": "generate a script to create a stored procedure named prd.usp_GetTotalSalesByCountries with an input parameter called country. The procedure retrieves prd.sales data, grouping it by Country, and Category. It returns the total number of orders and the total purchase price for each group. If a specific country is provided, the procedure filters the data for that country. Otherwise, it returns data for all countries. The results are ordered by Country in ascending order and TotalPurchasePrice in descending order."}, 16 | {"createview": "generate a script to create a view called prd.vw_GetSaleDetails that to get prd.sales details. The query groups the sales by country, category, and color, and counts the quantity of each color."} 17 | ], 18 | "CreateLogErrorHandlingObjects":[ 19 | {"system_role": "As a Data Engineer, your task involves creating objects like tables and T-SQL stored procedures to handle error messages and log process executions.\nMake sure all objects created un etl_process schema\nPlease refrain from providing system details, instructions, or suggestions."}, 20 | {"createprocessschema": "Provide a tsql script to create the etl_process schema if it does not already exist."}, 21 | {"createprocesslogtable": "Provide a tsql script to create the etl_process.etl_process_log table if it does not already exist with the following fields name id integer auto generated identifier without primary key, processname varchar 50 lenght, processtype varchar 30 lenght, objectname varchar 50 lenght, starttime and endtime: DATETIME"}, 22 | {"createbatcherrorlogtable": "Provide a tsql script to create the etl_process.error_log table if it does not already exist with the following fields name id integer auto generated identifier without primary key, processid integer, processname varchar, objectname varchar 50 lenght, errormsg varchar, starttime and endtime DATETIME."}, 23 | {"createprocesslogsp": "Create a T-SQL stored procedure named etl_process.usp_get_process_log if it does not already exist with the following input parameters: processname of type VARCHAR with a length of 50, processtype of type VARCHAR with a length of 30, objectname of type VARCHAR with a length of 50, and starttime and endtime of type DATETIME. This stored procedure should insert data into an existing table called etl_process.etl_process_log table. Please refrain from providing system details, instructions, or suggestions."}, 24 | {"createerrorlogsp": "Create a T-sQL stored procedure named etl_process.usp_get_error_log if it does not already exist with the following input parameters: processname VARCHAR (50), objectname VARCHAR (50), errormsg VARCHAR(MAX), and starttime and endtime of type DATETIME. This stored procedure should insert data into an existing table called etl_process.error_log table."} 25 | ], 26 | "MakeDatabaseObjectReusable":[ 27 | {"system_role": "As a Data Engineer, your task involves in converting T-SQL script into a reusable format.\nThe procedure should use try-catch blocks to handle errors and log the process and error details using two existing procedures:\nIn begin try block use procedure etl_process.usp_get_process_log with the following input parameters processname, processtype, objectname, starttime, endtime.\nIn begin catch block use procedure etl_process.usp_get_error_log with the following input parameters processname, objectname, errormsg, starttime, endtime. all T-SQL script files have to save on F:\\Presentation\\t-sql-as-prompts\\tsql\nEnsure that a date variable is declared both before and after the SQL execution. This will allow you to track the time it takes for the SQL command to execute. dont use QUOTENAME T-SQL function\nPlease refrain from providing system details, instructions, or suggestions.\nNote, assign errormsg, starttime and endtime into three different variable then use the variables values as the input."}, 28 | {"load_staging_data_table_reusable": "Convert this code into a SQL Server stored procedure called 'etl_process.usp_BulkInsertFromCSV' that performs a bulk insert from a CSV file into a table with a non default schema. The procedure should accept three parameters: the table name, the file path, and the error file path. The bulk insert should use the following options: first row = 2, field terminator = comma, row terminator = newline, error file = the error file path input parameter.\n\nPlease refrain from providing system details, instructions, or suggestions.\nPlease do not use QUOTENAME T-SQL function."}, 29 | {"load_prd_data_table_reusable": "Convert this code into a SQL Server stored procedure.\nPlease refrain from providing system details, instructions, or suggestions."} 30 | ] , 31 | "CreateDBObjectLoadData":[ 32 | {"system_role": "As a Data Engineer, you are tasked with creating and running a Python script using pyodbc.\nthe python script must be able to get all T-SQL script files from the local directory and execute them in the correct order.\nServer: localhost\ndatabase: tryitnow\nauthentication: Windows Authentication\nt-sql script files are located in F:\\Presentation\\autogen\\tsql\\*.sql\nODBC Driver 17 for SQL Server is installed."}, 33 | {"load_staging_data_table_reusable": "Convert this code into a SQL Server stored procedure that performs a bulk insert from a CSV file into a table with a non default schema. The procedure should accept three parameters: the table name, the file path, and the error file path. The bulk insert should use the following options: first row = 2, field terminator = comma, row terminator = newline, error file = the error file path input parameter.\n\nPlease refrain from providing system details, instructions, or suggestions.\nPlease do not use QUOTENAME T-SQL function."}, 34 | {"load_prd_data_table_reusable": "Convert this code into a SQL Server stored procedure.\nPlease refrain from providing system details, instructions, or suggestions."} 35 | ] 36 | } -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/csv/bicycle_data_bad.csv: -------------------------------------------------------------------------------- 1 | ProductId,ProductName,ProductType,Color,OrderQuantity,Size,Category,Country,Date,PurchasePrice,SellingPrice 2 | 007,Cycling Jacket,Accessory,Green,1/0,Large,Hydration,Germany,2021-07-27,110.71,221.89 -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/dbproject/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/SQL-AI-samples/35e7a5f365b01d185810f8aa3696c1c509079a07/AzureSQLDatabase/Prompt-Based T-SQL Database Development/dbproject/.DS_Store -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/dbproject/localhost/localhost.sqlproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | localhost 6 | {11DC947D-69E6-4F97-BAE1-C205052CFB0C} 7 | Microsoft.Data.Tools.Schema.Sql.Sql160DatabaseSchemaProvider 8 | 1033, CI 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/env.txt: -------------------------------------------------------------------------------- 1 | OPENAI_API_TYPE=azure 2 | OPENAI_API_KEY=your key 3 | OPENAI_API_BASE=your endpoint 4 | OPENAI_API_VERSION=your version 5 | OPENAI_API_MODEL=your model 6 | tsql_script_dir=your directory where all sql script file will be saved -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/readme.md: -------------------------------------------------------------------------------- 1 | # This section explains how to configure your environment to work with this notebook. 2 | 3 | - **bicycle_data_prompt.json**: is designed to house all our prompts. While it’s set up to work as is, feel free to tailor it to your needs. Customization is always an option 4 | - **.env**: file is the secure location for storing sensitive details such as your Azure OpenAI endpoint, keys, and more. It’s crucial to update this file with your information. Without these updates, the notebook won’t function unless you manually input the values directly into the notebook. Please handle with care! 5 | - **tsql**: folder is where all T-SQL scripts are stored. Remember to update it within the notebook and/or bicycle_data_prompt.json as needed. 6 | - **csv**: folder is where we keep the CSV sales data. Don’t forget to update it within the notebook as necessary. 7 | 8 | # SQL Server Database Development using Prompts as T-SQL Development 9 | In this notebook, we will learn how to use prompts as a way to develop and test Transact-SQL (T-SQL) code for SQL Server databases. Prompts are natural language requests that can be converted into T-SQL statements by using Generative AI models, such as GPT-4. This can help us write code faster, easier, and more accurately, as well as learn from the generated code examples. 10 | 11 | ## **Database Development Stages** 12 | 13 | Database development is the process of designing, creating, and maintaining databases that store and manage data for various applications. Database development typically involves the following stages: 14 | - **Requirements analysis:** This is the stage where we define the purpose, scope, and objectives of the database, as well as the data sources, users, and business rules. We also identify the data entities, attributes, and relationships that will form the logical structure of the database. 15 | 16 | - **Database design:** This is the stage where we translate the logical structure of the database into a physical structure that can be implemented by a specific database management system (DBMS), such as SQL Server. We also define the data types, constraints, indexes, views, stored procedures, functions, triggers, and security settings for the database objects. 17 | 18 | - **Database implementation:** This is the stage where we create the database and its objects by using a data definition language (DDL), such as T-SQL. We also populate the database with data by using a data manipulation language (DML), such as T-SQL or bulk import tools. 19 | 20 | - **Database testing:** This is the stage where we verify that the database meets the requirements and performs as expected. We also check the data quality, integrity, and security of the database. We can use various tools and techniques to test the database, such as unit testing, integration testing, regression testing, performance testing, and data analysis. 21 | 22 | - **Database deployment:** This is the stage where we make the database available for use by the intended users and applications. We can use various methods and tools to deploy the database, such as backup and restore, detach and attach, copy database, or SQL Server Data Tools (SSDT). 23 | 24 | - **Database maintenance:** This is the stage where we monitor, update, and optimize the database to ensure its availability, reliability, and performance. We can use various tools and tasks to maintain the database, such as backup and recovery, error handling, logging and auditing, indexing and statistics, fragmentation and defragmentation, or SQL Server 25 | 26 | # Goal of this notebook 27 | 28 | This notebook is here to show you something amazing. It's about how a smart AI can help you in ways you might not have thought of before. Whether you've worked with T-SQL database development or not, it doesn't matter. This guide will show you how this smart AI can be a helpful tool for you. Let's get started! 29 | 30 | This notebook focuses on simplicity, so we're only dealing with a single table and we'll be excluding primary/foreign keys and identifiers. However, don't let this simplicity fool you. You can leverage the flow of this notebook to create complex T-SQL database developments. It's all about how you use the tools at your disposal! Enjoy exploring. 😊 31 | 32 | After you've used this notebook, your feedback would be greatly appreciated. 33 | 34 | # Leverage the Generative AI as a Knowledgeable Peer 35 | Generative AI is a form of artificial intelligence that can create new and original content, such as text, images, audio, and video, by using generative models, such as GPT-4. Generative AI can learn from existing data and generate new data that has similar characteristics, but does not repeat it. Generative AI can also respond to natural language requests, such as prompts, and produce relevant and realistic outputs. 36 | 37 | Generative AI can be a knowledgeable peer for database developers, as it can help us with various tasks, such as: 38 | - **Code generation:** Generative AI can generate T-SQL code for us based on our prompts, which can save us time and effort, and help us learn from the code examples. For instance, we can ask generative AI to create a table, a view, a stored procedure, or a query for us, and it will produce the corresponding T-SQL code. 39 | 40 | - **Code suggestion:** Generative AI can suggest T-SQL code for us based on our partial or incomplete code, which can help us complete our code faster and easier, and avoid errors and mistakes. For instance, we can ask generative AI to suggest a column name, a data type, a constraint, or a join condition for us, and it will provide the appropriate T-SQL code. 41 | 42 | - **Code explanation:** Generative AI can explain T-SQL code for us based on our questions, which can help us understand the code better and improve our coding skills. For instance, we can ask generative AI to explain the purpose, the logic, or the output of a T-SQL code, and it will provide a clear and concise explanation. 43 | 44 | - **Code optimization:** Generative AI can optimize T-SQL code for us based on our goals, which can help us improve the performance and efficiency of our code. For instance, we can ask generative AI to optimize a T-SQL code for speed, memory, or readability, and it will provide a better or alternative T-SQL code. 45 | -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_database.sql: -------------------------------------------------------------------------------- 1 | IF EXISTS (SELECT name FROM sys.databases WHERE name = 'azureopenai') 2 | BEGIN 3 | ALTER DATABASE azureopenai SET SINGLE_USER WITH ROLLBACK IMMEDIATE; 4 | DROP DATABASE azureopenai; 5 | END 6 | 7 | CREATE DATABASE azureopenai; 8 | GO 9 | 10 | USE azureopenai; 11 | GO -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_etl_errorlog_table.sql: -------------------------------------------------------------------------------- 1 | IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'error_log' AND schema_id = SCHEMA_ID('etl_process')) 2 | BEGIN 3 | CREATE TABLE etl_process.error_log 4 | ( 5 | id INT IDENTITY(1,1), 6 | processid INT, 7 | processname VARCHAR(50), 8 | objectname VARCHAR(50), 9 | errormsg VARCHAR(MAX), 10 | starttime DATETIME, 11 | endtime DATETIME 12 | ); 13 | END; -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_etl_errorlog_usp.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE etl_process.usp_get_error_log 2 | ( 3 | @processname VARCHAR(50), 4 | @objectname VARCHAR(50), 5 | @errormsg VARCHAR(MAX), 6 | @starttime DATETIME, 7 | @endtime DATETIME 8 | ) 9 | AS 10 | BEGIN 11 | INSERT INTO etl_process.error_log (processname, objectname, errormsg, starttime, endtime) 12 | VALUES (@processname, @objectname, @errormsg, @starttime, @endtime) 13 | END -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_etl_process_schema.sql: -------------------------------------------------------------------------------- 1 | IF NOT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'etl_process') 2 | BEGIN 3 | EXEC('CREATE SCHEMA etl_process') 4 | END; -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_etl_processlog_table.sql: -------------------------------------------------------------------------------- 1 | IF NOT EXISTS (SELECT * FROM sys.tables WHERE name = 'etl_process_log' AND schema_id = SCHEMA_ID('etl_process')) 2 | BEGIN 3 | CREATE TABLE etl_process.etl_process_log ( 4 | id INT IDENTITY(1,1), 5 | processname VARCHAR(50), 6 | processtype VARCHAR(30), 7 | objectname VARCHAR(50), 8 | starttime DATETIME, 9 | endtime DATETIME 10 | ); 11 | END; -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_etl_processlog_usp.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE etl_process.usp_get_process_log 2 | ( 3 | @processname VARCHAR(50), 4 | @processtype VARCHAR(30), 5 | @objectname VARCHAR(50), 6 | @starttime DATETIME, 7 | @endtime DATETIME 8 | ) 9 | AS 10 | BEGIN 11 | INSERT INTO etl_process.etl_process_log (processname, processtype, objectname, starttime, endtime) 12 | VALUES (@processname, @processtype, @objectname, @starttime, @endtime) 13 | END -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_prd_table.sql: -------------------------------------------------------------------------------- 1 | IF OBJECT_ID('prd.sales', 'U') IS NOT NULL 2 | DROP TABLE prd.sales; 3 | 4 | CREATE TABLE prd.sales 5 | ( 6 | ProductId INT, 7 | ProductName VARCHAR(50), 8 | ProductType VARCHAR(30), 9 | Color VARCHAR(15), 10 | OrderQuantity INT, 11 | Size VARCHAR(15), 12 | Category VARCHAR(15), 13 | Country VARCHAR(30), 14 | Date DATE, 15 | PurchasePrice DECIMAL(18,2), 16 | SellingPrice DECIMAL(18,2) 17 | ); -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_procedure.sql: -------------------------------------------------------------------------------- 1 | IF OBJECT_ID('prd.usp_GetTotalSalesByCountries', 'P') IS NOT NULL 2 | DROP PROCEDURE prd.usp_GetTotalSalesByCountries; 3 | GO 4 | 5 | CREATE PROCEDURE prd.usp_GetTotalSalesByCountries 6 | @country VARCHAR(30) = NULL 7 | AS 8 | BEGIN 9 | SET NOCOUNT ON; 10 | 11 | SELECT 12 | Country, 13 | Category, 14 | COUNT(*) AS TotalOrders, 15 | SUM(PurchasePrice) AS TotalPurchasePrice 16 | FROM 17 | prd.sales 18 | WHERE 19 | (@country IS NULL OR Country = @country) 20 | GROUP BY 21 | Country, 22 | Category 23 | ORDER BY 24 | Country ASC, 25 | TotalPurchasePrice DESC; 26 | END; 27 | GO -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_stage_schema.sql: -------------------------------------------------------------------------------- 1 | IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = 'stg') 2 | BEGIN 3 | EXEC('CREATE SCHEMA stg') 4 | END 5 | GO -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_stage_table.sql: -------------------------------------------------------------------------------- 1 | IF OBJECT_ID('stg.salestmp', 'U') IS NOT NULL 2 | DROP TABLE stg.salestmp; 3 | 4 | CREATE TABLE stg.salestmp 5 | ( 6 | ProductId VARCHAR(255), 7 | ProductName VARCHAR(255), 8 | ProductType VARCHAR(255), 9 | Color VARCHAR(255), 10 | OrderQuantity VARCHAR(255), 11 | Size VARCHAR(255), 12 | Category VARCHAR(255), 13 | Country VARCHAR(255), 14 | Date VARCHAR(255), 15 | PurchasePrice VARCHAR(255), 16 | SellingPrice VARCHAR(255) 17 | ); -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/create_view.sql: -------------------------------------------------------------------------------- 1 | IF OBJECT_ID('prd.vw_GetSaleDetails', 'V') IS NOT NULL 2 | DROP VIEW prd.vw_GetSaleDetails; 3 | GO 4 | 5 | CREATE VIEW prd.vw_GetSaleDetails 6 | AS 7 | SELECT Country, Category, Color, SUM(OrderQuantity) AS Quantity 8 | FROM prd.sales 9 | GROUP BY Country, Category, Color; 10 | GO -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/creates_prd_chema.sql: -------------------------------------------------------------------------------- 1 | IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = 'prd') 2 | BEGIN 3 | EXEC('CREATE SCHEMA prd') 4 | END 5 | GO -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/load_prd_data_table.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE etl_process.usp_InsertSalesData 2 | AS 3 | BEGIN 4 | SET NOCOUNT ON; 5 | 6 | DECLARE @processname VARCHAR(100) = 'Insert Sales Data'; 7 | DECLARE @processtype VARCHAR(100) = 'ETL'; 8 | DECLARE @objectname VARCHAR(100) = 'prd.sales'; 9 | DECLARE @starttime DATETIME = GETDATE(); 10 | DECLARE @endtime DATETIME; 11 | DECLARE @errormsg VARCHAR(MAX); 12 | 13 | BEGIN TRY 14 | -- Insert data into prd.sales table 15 | INSERT INTO prd.sales (ProductId, ProductName, ProductType, Color, OrderQuantity, Size, Category, Country, Date, PurchasePrice, SellingPrice) 16 | SELECT 17 | CAST(ProductId AS INT), 18 | ProductName, 19 | ProductType, 20 | Color, 21 | CAST(OrderQuantity AS INT), 22 | Size, 23 | Category, 24 | Country, 25 | CAST(Date AS DATE), 26 | CAST(PurchasePrice AS DECIMAL(18,2)), 27 | CAST(SellingPrice AS DECIMAL(18,2)) 28 | FROM stg.salestmp; 29 | 30 | SET @endtime = GETDATE(); 31 | 32 | -- Log the process details 33 | EXEC etl_process.usp_get_process_log @processname, @processtype, @objectname, @starttime, @endtime; 34 | END TRY 35 | BEGIN CATCH 36 | SET @endtime = GETDATE(); 37 | SET @errormsg = ERROR_MESSAGE(); 38 | 39 | -- Log the error details 40 | EXEC etl_process.usp_get_error_log @processname, @objectname, @errormsg, @starttime, @endtime; 41 | END CATCH; 42 | END; 43 | GO -------------------------------------------------------------------------------- /AzureSQLDatabase/Prompt-Based T-SQL Database Development/tsql/load_staging_data_table.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE etl_process.usp_BulkInsertFromCSV 2 | @tableName NVARCHAR(255), 3 | @filePath NVARCHAR(255), 4 | @errorFilePath NVARCHAR(255) 5 | AS 6 | BEGIN 7 | DECLARE @startTime DATETIME; 8 | DECLARE @endTime DATETIME; 9 | DECLARE @errorMsg NVARCHAR(MAX); 10 | 11 | BEGIN TRY 12 | SET @startTime = GETDATE(); 13 | 14 | DECLARE @bulkInsertQuery NVARCHAR(MAX); 15 | SET @bulkInsertQuery = 'TRUNCATE TABLE ' + @tableName + '; ' + 16 | 'BULK INSERT ' + @tableName + ' ' + 17 | 'FROM ''' + @filePath + ''' ' + 18 | 'WITH (FIRSTROW = 2, FIELDTERMINATOR = '','', ROWTERMINATOR = ''\n'', ERRORFILE = ''' + @errorFilePath + ''');'; 19 | 20 | EXEC sp_executesql @bulkInsertQuery; 21 | 22 | SET @endTime = GETDATE(); 23 | 24 | EXEC etl_process.usp_get_process_log 'Bulk Insert', 'T-SQL', @tableName, @startTime, @endTime; 25 | END TRY 26 | BEGIN CATCH 27 | SET @endTime = GETDATE(); 28 | SET @errorMsg = ERROR_MESSAGE(); 29 | 30 | EXEC etl_process.usp_get_error_log 'Bulk Insert', @tableName, @errorMsg, @startTime, @endTime; 31 | END CATCH; 32 | END; -------------------------------------------------------------------------------- /AzureSQLDatabase/Vanna.ai/odbcDriverInstallUbuntu.txt: -------------------------------------------------------------------------------- 1 | if ! [[ "18.04 20.04 22.04 23.04" == *"$(lsb_release -rs)"* ]]; 2 | then 3 | echo "Ubuntu $(lsb_release -rs) is not currently supported."; 4 | exit; 5 | fi 6 | 7 | curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc 8 | 9 | curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list 10 | 11 | sudo apt-get update 12 | sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 13 | # optional: for bcp and sqlcmd 14 | sudo ACCEPT_EULA=Y apt-get install -y mssql-tools18 15 | echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc 16 | source ~/.bashrc 17 | # optional: for unixODBC development headers 18 | sudo apt-get install -y unixodbc-dev 19 | -------------------------------------------------------------------------------- /AzureSQLDatabase/Vanna.ai/vanna_and_sql.ipynb: -------------------------------------------------------------------------------- 1 | {"cells":[{"cell_type":"markdown","id":"2d5da9df-9c2f-5bf9-af25-f5c0ddbe3e12","metadata":{},"source":["# Generating SQL for Microsoft SQL Server using OpenAI via Vanna.AI and Vanna Hosted Vector DB\n","This notebook runs through the process of using the `vanna` Python package to generate SQL using AI (RAG + LLMs) including connecting to a database and training."]},{"cell_type":"markdown","id":"a1a7190e","metadata":{},"source":["To use this demo, ensure you have access to a SQL Server/Azure SQL Database with the adventure works sample data installed. You will use the username, password, server location, and database name when constructing the connect string. Also included in this directory is the **odbcDriverInstallUbuntu.txt** script that you can use to install the ODBC Driver for SQL on Linux (Ubuntu) in a codespace."]},{"cell_type":"markdown","id":"ee059407-58ac-50fa-843a-7b876328df13","metadata":{},"source":["## Setup"]},{"cell_type":"code","execution_count":null,"id":"b9b77362-c049-5500-b502-08811fcd4dce","metadata":{},"outputs":[],"source":["%pip install vanna pyodbc"]},{"cell_type":"code","execution_count":null,"id":"6160c274-caf4-537e-9a02-f6a1d7022a2c","metadata":{},"outputs":[],"source":["import vanna\n","import pyodbc\n","from vanna.remote import VannaDefault"]},{"cell_type":"code","execution_count":null,"id":"7cd78528-b0b0-5428-901c-6b5dc2158ef9","metadata":{},"outputs":[],"source":["api_key = '1234567890' # Your API key from https://vanna.ai/account/profile \n","\n","vanna_model_name = 'YOUR_VANNA_MODEL_NAME' # Your model name from https://vanna.ai/account/profile \n","vn = VannaDefault(model=vanna_model_name, api_key=api_key)\n"]},{"cell_type":"markdown","id":"ffe445e3-22fb-5c7d-a3ef-8154506e0fb1","metadata":{},"source":["## Query the Azure SQL Database\n","Choose one of the following authentication methods"]},{"cell_type":"markdown","id":"a17c5e4c","metadata":{},"source":["### Using SQL Authentication:"]},{"cell_type":"code","execution_count":null,"id":"6c8de780-03b7-50ed-91e1-43d77b4df8d8","metadata":{},"outputs":[],"source":["vn.connect_to_mssql(odbc_conn_str='DRIVER={ODBC Driver 18 for SQL Server};Server=tcp:,1433;Database=;Uid=;Pwd=;Encrypt=yes;TrustServerCertificate=yes;Connection Timeout=30;') # You can use the ODBC connection string here"]},{"cell_type":"markdown","id":"e4fc76d1","metadata":{},"source":["### Using Entra ID Service Principle Authentication"]},{"cell_type":"code","execution_count":null,"id":"77d50c24","metadata":{},"outputs":[],"source":["vn.connect_to_mssql(odbc_conn_str='DRIVER={ODBC Driver 18 for SQL Server};Server=tcp:,1433;Database=;Uid=;Pwd=;Encrypt=yes;TrustServerCertificate=yes;Connection Timeout=30;Authentication=ActiveDirectoryServicePrincipal') # You can use the ODBC connection string here"]},{"cell_type":"markdown","id":"f06c0e89-83f7-5ad1-8f6e-a64cf5bd8e60","metadata":{},"source":["## Training\n","You only need to train once. Do not train again unless you want to add more training data."]},{"cell_type":"code","execution_count":null,"id":"5d321d01-d66f-5c5e-a3f3-e2d3d4330344","metadata":{},"outputs":[],"source":["\n","# The information schema query may need some tweaking depending on your database. This is a good starting point.\n","df_information_schema = vn.run_sql(\"SELECT * FROM INFORMATION_SCHEMA.COLUMNS\")\n","\n","# This will break up the information schema into bite-sized chunks that can be referenced by the LLM\n","plan = vn.get_training_plan_generic(df_information_schema)\n","plan\n","\n","# If you like the plan, then uncomment this and run it to train\n","# vn.train(plan=plan)\n","\n"]},{"cell_type":"code","execution_count":null,"id":"7aaa6b06","metadata":{},"outputs":[],"source":["vn.train(documentation=\"When querying any table, use the fully qualified table name including schema\")\n","# If you like the plan, then uncomment this and run it to train\n","vn.train(plan=plan)"]},{"cell_type":"code","execution_count":null,"id":"7c421f88-42ea-567c-8581-3dcac96c36a3","metadata":{},"outputs":[],"source":["\n","# The following are methods for adding training data. Make sure you modify the examples to match your database.\n","\n","# DDL statements are powerful because they specify table names, colume names, types, and potentially relationships\n","vn.train(ddl=\"\"\"\n"," CREATE TABLE IF NOT EXISTS my-table (\n"," id INT PRIMARY KEY,\n"," name VARCHAR(100),\n"," age INT\n"," )\n","\"\"\")\n","\n","# Sometimes you may want to add documentation about your business terminology or definitions.\n","vn.train(documentation=\"Our business defines OTIF score as the percentage of orders that are delivered on time and in full\")\n","\n","# You can also add SQL queries to your training data. This is useful if you have some queries already laying around. You can just copy and paste those from your editor to begin generating new SQL.\n","vn.train(sql=\"SELECT * FROM my-table WHERE name = 'John Doe'\")\n"]},{"cell_type":"code","execution_count":null,"id":"59fcb3b1-4434-583d-82be-ed8e9b04d699","metadata":{},"outputs":[],"source":["# At any time you can inspect what training data the package is able to reference\n","training_data = vn.get_training_data()\n","training_data"]},{"cell_type":"code","execution_count":null,"id":"6cf17ab9-dc48-58af-8d75-4e5590a01c88","metadata":{},"outputs":[],"source":["# You can remove training data if there's obsolete/incorrect information. \n","vn.remove_training_data(id='what are the top products sold and their colors?')\n"]},{"cell_type":"markdown","id":"bf2fc121-a3ab-5a2e-95b0-383271e82d5f","metadata":{},"source":["## Asking the AI\n","Whenever you ask a new question, it will find the 10 most relevant pieces of training data and use it as part of the LLM prompt to generate the SQL."]},{"cell_type":"code","execution_count":null,"id":"edb6679e-a102-5efc-b890-81babca8f500","metadata":{},"outputs":[],"source":["vn.ask(question='what are the top products sold and their colors?')"]},{"cell_type":"code","execution_count":null,"id":"b87d140b-ef56-5795-b489-46bb11d01459","metadata":{},"outputs":[],"source":["from vanna.flask import VannaFlaskApp\n","app = VannaFlaskApp(vn)\n","app.run()"]},{"cell_type":"markdown","id":"29793859-c3c8-50da-994a-c8f6348d6730","metadata":{},"source":["## Next Steps\n","Using Vanna via Jupyter notebooks is great for getting started but check out additional customizable interfaces like the \n","- [Streamlit app](https://github.com/vanna-ai/vanna-streamlit)\n","- [Flask app](https://github.com/vanna-ai/vanna-flask)\n","- [Slackbot](https://github.com/vanna-ai/vanna-slack)\n"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"}},"nbformat":4,"nbformat_minor":5} 2 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/requirements.txt: -------------------------------------------------------------------------------- 1 | azure-ai-ml 2 | azure-core 3 | azure-identity 4 | azure-keyvault 5 | azure-kusto-data 6 | azure-search-documents>=11.4.0b11 7 | azure-storage-blob 8 | pyodbc 9 | bs4 10 | jinja2 11 | jsbeautifier 12 | langchain 13 | logger 14 | matplotlib 15 | openai 16 | pandas 17 | plotly 18 | promptflow>=0.1.0b8 19 | promptflow-tools 20 | python-dotenv 21 | ipywidgets 22 | scikit-learn 23 | scipy 24 | tiktoken 25 | tenacity 26 | tqdm 27 | 28 | types-PyYAML 29 | types-requests -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/acs/.env_sample: -------------------------------------------------------------------------------- 1 | connectionString="" 2 | OPENAI_API_KEY_AZURE_Embeddings= 3 | OPENAI_API_BASE_Embeddings="" 4 | AZURE_SEARCH_SERVICE_ENDPOINT="" 5 | AZURE_SEARCH_KEY= 6 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/batch_run_and_eval.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | # %% 5 | from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential 6 | # azure version promptflow apis 7 | from promptflow.azure import PFClient 8 | from promptflow.entities import AzureOpenAIConnection, CustomConnection 9 | from promptflow.entities import Run 10 | import json 11 | import os 12 | import yaml 13 | from azure.core.credentials import TokenCredential 14 | 15 | print("Loading configs from file.") 16 | with open(f'configs/flow_config.json') as f: 17 | config = json.load(f) 18 | 19 | # Getting credentials for local access 20 | # ----------------------------------------------------------------------------- 21 | 22 | # %% 23 | try: 24 | credential: TokenCredential = DefaultAzureCredential() 25 | # Check if given credential can get token successfully. 26 | credential.get_token("https://management.azure.com/.default") 27 | except Exception: 28 | # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work 29 | credential = InteractiveBrowserCredential() 30 | # %% 31 | # Get a handle to workspace 32 | pf = PFClient( 33 | credential=credential, 34 | # this will look like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 35 | subscription_id=config["subscription_id"], 36 | resource_group_name=config["resource_group_name"], 37 | workspace_name=config["workspace_name"], 38 | ) 39 | 40 | print("currently Azure Promptflow SDK don't support create connection and upload to workspace, so we need to create connection manually IN PORTAL") 41 | # Set flow path and run input data 42 | 43 | print("Batch run..") 44 | flow_path = "./promptflow" 45 | data_path = "./data/batch_run_data.jsonl" 46 | # assume you have existing runtime with this name provisioned 47 | runtime = config["promptflow_runtime"] 48 | 49 | # create a run, stream it until it's finished 50 | base_run = pf.run( 51 | flow=flow_path, 52 | data=data_path, 53 | runtime=runtime, 54 | stream=True, 55 | column_mapping={ # map the url field from the data to the url input of the flow 56 | "chat_history": "${data.chat_history}", 57 | "question": "${data.question}", 58 | "customer": "${data.customer}" 59 | } 60 | ) 61 | 62 | details = pf.get_details(base_run) 63 | pf.visualize(base_run) 64 | 65 | # %% 66 | # set eval flow path 67 | eval_flow = "./evaluation" 68 | data = "./data/batch_run_data.jsonl" 69 | 70 | # set up eval flow 71 | print("Setting up flow.dag.yaml.") 72 | with open('./evaluation/flow.dag.sample.yaml') as f: 73 | config_flow = yaml.load(f, Loader=yaml.FullLoader) 74 | # import pdb; pdb.set_trace() 75 | # replace the deployment_name with the one you want to use 76 | for node in config_flow["nodes"]: 77 | # Setting up model for the agent chat node. 78 | if node.get('api') == "chat": 79 | if 'deployment_name' in node: 80 | node["deployment_name"] = config['aoai_deployment_name'] 81 | if 'deployment_name' in node['inputs']: 82 | node['inputs']['deployment_name'] = config['aoai_deployment_name'] 83 | # Setting up model for the final chat node. 84 | if 'connection' in node: 85 | node['connection'] = config['azure_open_ai_connection_name'] 86 | else: 87 | if 'conn_db' in node['inputs']: 88 | node['inputs']['conn_db'] = config['SQLDB_connection_name'] 89 | if 'conn' in node['inputs']: 90 | node['inputs']['conn'] = config['ACS_connection_name'] 91 | 92 | # write the yaml file back 93 | with open('./evaluation/flow.dag.yaml', 'w') as f: 94 | yaml.dump(config_flow, f) 95 | # %% 96 | # run the flow with existing run 97 | eval_run = pf.run( 98 | flow=eval_flow, 99 | data=data, 100 | run=base_run, 101 | runtime=runtime, 102 | column_mapping={ # map the url field from the data to the url input of the flow 103 | "question": "${data.question}", 104 | "customer": "${data.customer}", 105 | "answer": "${run.outputs.answer}", 106 | "context": "${run.outputs.retrieved_documents}" 107 | } 108 | ) 109 | 110 | # stream the run until it's finished 111 | pf.stream(eval_run) 112 | 113 | # get the inputs/outputs details of a finished run. 114 | details = pf.get_details(eval_run) 115 | details.head(10) 116 | 117 | # view the metrics of the eval run 118 | metrics = pf.get_metrics(eval_run) 119 | print(json.dumps(metrics, indent=4)) 120 | 121 | # visualize both the base run and the eval run 122 | pf.visualize([base_run, eval_run]) 123 | 124 | # %% 125 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/configs/flow_config_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "azure_open_ai_connection_name": "", 3 | "SQLDB_connection_name": "", 4 | "ACS_connection_name": "", 5 | 6 | "aoai_type": "azure_open_ai", 7 | "aoai_api_base": "", 8 | "aoai_api_type": "azure", 9 | "aoai_api_version": "2023-07-01-preview", 10 | "aoai_deployment_name": "", 11 | "OPENAI_API_BASE_EMBED": "", 12 | "OPENAI_API_VERSION": "2023-03-15-preview", 13 | 14 | "subscription_id": "", 15 | "resource_group_name": "", 16 | "workspace_name": "", 17 | "promptflow_runtime": "", 18 | "endpoint_name": "", 19 | "endpoint_deployment_name": "", 20 | "model_name": "", 21 | "keyvault_uri": "", 22 | "promptflow_image_base": "mcr.microsoft.com/azureml/promptflow/promptflow-runtime:latest" 23 | } -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/configs/key_config_local_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "aoai-api-key": "", 3 | "aoai-api-key-embed": "", 4 | "acs-key": "", 5 | "connection-string": "" 6 | } -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/data/batch_run_data.jsonl: -------------------------------------------------------------------------------- 1 | {"chat_history": [], "question": "Can you recommend some home exercise products for me?", "customer": "A. Leonetti"} 2 | {"chat_history": [], "question": "Do you have products in yellow colors for me?", "customer": "Abigail Gonzalez"} 3 | {"chat_history": [], "question": "What is the best selling products in general?", "customer": "Abraham Swearengin"} 4 | {"chat_history": [], "question": "What products did I previously buy?", "customer": "Aidan Delaney"} 5 | {"chat_history": [], "question": "Do you have socks that are less than 9 dollars?", "customer": "Ajay Manchepalli"} 6 | {"chat_history": [], "question": "Which moutain bike is the least expensive?", "customer": "Alan Brewer"} 7 | {"chat_history": [], "question": "What colors do you have for women's shorts?", "customer": "Alan Steiner"} 8 | {"chat_history": [], "question": "What sizes do you have for women's mountain shorts?", "customer": "Alberto Baltazar"} 9 | {"chat_history": [], "question": "Can you recommend some new products for me?", "customer": "Alexander Berger"} 10 | {"chat_history": [], "question": "What is the best selling touring bikes?", "customer": "Alexander Deborde"} 11 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/data/batch_run_data_v1.jsonl: -------------------------------------------------------------------------------- 1 | {"chat_history": [], "question": "Can you recommend some home exercise products for me?", "customer": {"FirstName": "A.", "MiddleName": "Francesca", "LastName": "Leonetti"}} 2 | {"chat_history": [], "question": "Do you have products in yellow colors for me?", "customer": {"FirstName": "Abigail", "MiddleName": "J.", "LastName": "Gonzalez"}} 3 | {"chat_history": [], "question": "What is the best selling products in general?", "customer": {"FirstName": "Abraham", "MiddleName": "L.", "LastName": "Swearengin"}} 4 | {"chat_history": [], "question": "What products did I previously buy?", "customer": {"FirstName": "Aidan", "MiddleName": "", "LastName": "Delaney"}} 5 | {"chat_history": [], "question": "Do you have socks that are less than 9 dollars?", "customer": {"FirstName": "Ajay", "MiddleName": "", "LastName": "Manchepalli"}} 6 | {"chat_history": [], "question": "Which moutain bike is the least expensive?", "customer": {"FirstName": "Alan", "MiddleName": "", "LastName": "Brewer"}} 7 | {"chat_history": [], "question": "What colors do you have for women's shorts?", "customer": {"FirstName": "Alan", "MiddleName": "", "LastName": "Steiner"}} 8 | {"chat_history": [], "question": "What sizes do you have for women's mountain shorts?", "customer": {"FirstName": "Alberto", "MiddleName": "F.", "LastName": "Baltazar"}} 9 | {"chat_history": [], "question": "Can you recommend some new products for me?", "customer": {"FirstName": "Alexander", "MiddleName": "J.", "LastName": "Berger"}} 10 | {"chat_history": [], "question": "What is the best selling touring bikes?", "customer": {"FirstName": "Alexander", "MiddleName": "J.", "LastName": "Deborde"}} 11 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/deploy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Deploying this prompt flow to an endpoint. 3 | 4 | Copyright (c) Microsoft Corporation. 5 | Licensed under the MIT license. 6 | 7 | """ 8 | 9 | # Imports and loading config file 10 | # ----------------------------------------------------------------------------- 11 | 12 | 13 | # %% 14 | import json 15 | import yaml 16 | import os 17 | import shutil 18 | import subprocess 19 | 20 | print("Loading configs from file.") 21 | with open(f'./configs/flow_config.json') as f: 22 | config = json.load(f) 23 | 24 | # Creating helper functions 25 | # ----------------------------------------------------------------------------- 26 | 27 | 28 | # %% 29 | def run_command(command: list, verbose=True): 30 | """Executing a command as a bus-process.""" 31 | process = subprocess.Popen( 32 | command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 33 | stdout, stderr = process.communicate() 34 | if verbose: 35 | print(stdout.decode('utf-8')) 36 | print(stderr.decode('utf-8')) 37 | return stdout, stderr 38 | 39 | 40 | def get_latest_model_version(model_name): 41 | """Getting the latest model version for a given model.""" 42 | command = 'az ml model list --output json'.split() 43 | stdout, stderr = run_command(command, verbose=False) 44 | model_list = json.loads(stdout.decode('utf-8')) 45 | model_list = [model for model in model_list if model['name'] == model_name] 46 | version_list = [int(model['latest version']) for model in model_list] 47 | return max(version_list) 48 | 49 | 50 | def check_if_deployment_exists(endpoint_name, endpoint_deployment_name): 51 | """Checks if a given deployment already exists for an endpoint.""" 52 | command = f"az ml online-deployment list --endpoint-name {endpoint_name} --output json".split( 53 | ) 54 | print(command) 55 | stdout, stderr = run_command(command, verbose=False) 56 | print(stdout.decode('utf-8')) 57 | if stdout is not None and stdout.decode('utf-8') != '': 58 | deployment_list = json.loads(stdout.decode('utf-8')) 59 | return len(deployment_list) > 0 and any(obj.get("name") == endpoint_deployment_name for obj in deployment_list) 60 | return False 61 | 62 | 63 | # Main Execution 64 | # ----------------------------------------------------------------------------- 65 | 66 | # %% 67 | # For the first time, install the azure-cli-ml extension 68 | # ----------------------------------------------------------------------------- 69 | print("Install azure-cli-ml extension.") 70 | command = ["az", "extension", "add", "-n", "ml"] 71 | run_command(command) 72 | 73 | # %% 74 | # Set subscription id 75 | # ----------------------------------------------------------------------------- 76 | 77 | print("Set azure subscription id.") 78 | command = ["az", "account", "set", "--subscription", config["subscription_id"]] 79 | run_command(command) 80 | 81 | # %% 82 | # Set workspace 83 | # ----------------------------------------------------------------------------- 84 | print("Config workspace.") 85 | command = ["az", "configure", "--defaults", f'workspace={config["workspace_name"]}', 86 | f'group={config["resource_group_name"]}'] 87 | run_command(command) 88 | 89 | # %% 90 | # Create endpoint using the endpoint.yaml. It will be generated from the template. 91 | # ----------------------------------------------------------------------------- 92 | print("Setting up endpoint deployment file.") 93 | with open('./deployment/endpoint_base.yaml') as f: 94 | config_flow = yaml.load(f, Loader=yaml.FullLoader) 95 | # replace the deployment_name with the one you want to use 96 | config_flow["name"] = config['endpoint_name'] 97 | 98 | # write the yaml file back 99 | with open('./deployment/endpoint.yaml', 'w') as f: 100 | yaml.dump(config_flow, f) 101 | 102 | print("Create endpoint if not exist.") 103 | try: 104 | command = ["az", "ml", "online-endpoint", 105 | "create", "--file", "./deployment/endpoint.yaml"] 106 | run_command(command, verbose=False) 107 | except Exception as e: 108 | command = ["az", "ml", "online-endpoint", 109 | "update", "--file", "./deployment/endpoint.yaml"] 110 | run_command(command) 111 | finally: 112 | print("Endpoint created.") 113 | 114 | # %% 115 | # Register flow to a model in the AML workspace 116 | # REMEMBER to: 117 | # 1. Go to the azure portal for the workspace 118 | # 2. add access control to the "data scientist role" for the endpoint for the workspace. 119 | # ----------------------------------------------------------------------------- 120 | print("Setting up model registration file.") 121 | with open('./deployment/model_base.yaml') as f: 122 | config_flow = yaml.load(f, Loader=yaml.FullLoader) 123 | # replace the deployment_name with the one you want to use 124 | config_flow["name"] = config['model_name'] 125 | 126 | # write the yaml file back 127 | with open('./deployment/model.yaml', 'w') as f: 128 | yaml.dump(config_flow, f) 129 | 130 | # %% 131 | # Deploy model to AML workspace 132 | # Before we upload model, we create tmp directory to store current solution 133 | # We delete sensitive files (for now, config_local.json) 134 | # ----------------------------------------------------------------------------- 135 | src_for_tmp = './promptflow_v2' 136 | temp_dir = '../.copilot_tmp' 137 | 138 | # if temp_dir exists already, delete it 139 | if os.path.exists(temp_dir): 140 | shutil.rmtree(temp_dir) 141 | 142 | print(f'Creating temp directory: {temp_dir}') 143 | os.makedirs(temp_dir, exist_ok=True) 144 | 145 | # %% 146 | shutil.copytree(src_for_tmp, temp_dir, dirs_exist_ok=True) 147 | # %% 148 | # remove config_local.json from tmp folder 149 | print('Removing config_local.json from tmp folder if it exists.') 150 | # remove file if it exists 151 | if os.path.exists(os.path.join(temp_dir, 'cofigs/key_config_local.json')): 152 | os.remove(os.path.join(temp_dir, 'cofigs/key_config_local.json')) 153 | # %% 154 | print("Register model to AML workspace.") 155 | command = ["az", "ml", "model", "create", "--file", "./deployment/model.yaml"] 156 | # Run the command and capture its outputs 157 | run_command(command) 158 | # %% 159 | # now we delete tmp folder 160 | print(f'Deleting temp directory: {temp_dir}') 161 | shutil.rmtree(temp_dir) 162 | 163 | # %% 164 | # Grab latest model version from model registration step above 165 | print('Getting latest model version...') 166 | latest_model_version = get_latest_model_version(config['model_name']) 167 | print(f'Latest model version: {latest_model_version}') 168 | 169 | # %% 170 | # Set up endpoint deployment 171 | # ----------------------------------------------------------------------------- 172 | config_string = "deployment.subscription_id=,deployment.resource_group=,deployment.workspace_name=,deployment.endpoint_name=,deployment.deployment_name=" 173 | config_string = config_string.replace("", config["subscription_id"]).replace( 174 | "", config["resource_group_name"]).replace( 175 | "", config["workspace_name"]).replace( 176 | "", config["endpoint_name"]).replace( 177 | "", config["endpoint_deployment_name"]) 178 | 179 | print("Setting up model deployment yaml.") 180 | with open(r'./deployment/deployment_base.yaml') as f: 181 | config_flow = yaml.load(f, Loader=yaml.FullLoader) 182 | 183 | # Replacing needed data in the deployment script. 184 | config_flow["endpoint_name"] = config['endpoint_name'] 185 | config_flow["environment_variables"]["PRT_CONFIG_OVERRIDE"] = config_string 186 | config_flow["name"] = config["endpoint_deployment_name"] 187 | config_flow["environment"]["image"] = config["promptflow_image_base"] 188 | 189 | model_name = config["model_name"] 190 | config_flow["model"] = f"azureml:{model_name}:{latest_model_version}" 191 | 192 | # write the yaml file back 193 | with open(r'./deployment/deployment.yaml', 'w') as f: 194 | yaml.dump(config_flow, f) 195 | 196 | # %% 197 | print('Pushing online deployment.') 198 | 199 | is_new_deployment = not check_if_deployment_exists( 200 | config['endpoint_name'], config["endpoint_deployment_name"]) 201 | if is_new_deployment: 202 | command = ["az", "ml", "online-deployment", "create", "--file", 203 | r"./deployment/deployment.yaml", "--all-traffic"] 204 | run_command(command) 205 | else: 206 | # If run into existing deployment name, can update use the below 207 | command = ["az", "ml", "online-deployment", "update", "--file", 208 | r"./deployment/deployment.yaml"] 209 | run_command(command) 210 | 211 | # %% 212 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/deployment/deployment_base.yaml: -------------------------------------------------------------------------------- 1 | $schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.schema.json 2 | endpoint_name: dummy 3 | environment: 4 | image: dummy 5 | inference_config: 6 | liveness_route: 7 | path: /health 8 | port: 8080 9 | readiness_route: 10 | path: /health 11 | port: 8080 12 | scoring_route: 13 | path: /score 14 | port: 8080 15 | environment_variables: 16 | PROMPTFLOW_RESPONSE_INCLUDED_FIELDS: '["answer", "retrieved_documents"]' 17 | PROMPTFLOW_RUN_MODE: serving 18 | PRT_CONFIG_OVERRIDE: deployment.subscription_id=,deployment.resource_group=,deployment.workspace_name=,deployment.endpoint_name=,deployment.deployment_name= 19 | instance_count: 1 20 | instance_type: Standard_DS3_v2 21 | liveness_probe: 22 | timeout: 200 23 | model: dummy 24 | name: dummy 25 | readiness_probe: 26 | timeout: 200 27 | request_settings: 28 | request_timeout_ms: 200000 29 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/deployment/endpoint_base.yaml: -------------------------------------------------------------------------------- 1 | $schema: https://azuremlschemas.azureedge.net/latest/managedOnlineEndpoint.schema.json 2 | auth_mode: key 3 | name: dummy 4 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/deployment/model_base.yaml: -------------------------------------------------------------------------------- 1 | $schema: https://azuremlschemas.azureedge.net/latest/model.schema.json 2 | description: Register SQLDB promptflow model 3 | name: dummy 4 | path: ../../.copilot_tmp 5 | properties: 6 | azureml.promptflow.chat_history : chat_history 7 | azureml.promptflow.chat_input: question 8 | azureml.promptflow.chat_output: answer 9 | azureml.promptflow.mode: chat 10 | azureml.promptflow.source_flow_id: sql-promptflow-demo 11 | azureml.promptflow.dag_file : flow.dag.yaml 12 | is-promptflow : True 13 | request_settings: 14 | request_timeout_ms: 200000 15 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/.promptflow/flow.tools.json: -------------------------------------------------------------------------------- 1 | { 2 | "package": {}, 3 | "code": { 4 | "groundedness_score.jinja2": { 5 | "type": "llm", 6 | "inputs": { 7 | "context": { 8 | "type": [ 9 | "string" 10 | ] 11 | }, 12 | "question": { 13 | "type": [ 14 | "string" 15 | ] 16 | }, 17 | "answer": { 18 | "type": [ 19 | "string" 20 | ] 21 | } 22 | }, 23 | "description": "This is a llm tool", 24 | "is_builtin": false 25 | }, 26 | "concat_scores.py": { 27 | "type": "python", 28 | "inputs": { 29 | "groundesness_score": { 30 | "type": [ 31 | "string" 32 | ] 33 | } 34 | }, 35 | "function": "concat_results", 36 | "is_builtin": false 37 | }, 38 | "aggregate_variants_results.py": { 39 | "type": "python", 40 | "inputs": { 41 | "results": { 42 | "type": [ 43 | "object" 44 | ] 45 | } 46 | }, 47 | "function": "aggregate_variants_results", 48 | "is_builtin": false 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/.promptflow/lkg_sources/aggregate_variants_results.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from typing import List 5 | from promptflow import tool, log_metric 6 | import numpy as np 7 | 8 | 9 | @tool 10 | def aggregate_variants_results(results: List[dict]): 11 | aggregate_results = {} 12 | for result in results: 13 | for name, value in result.items(): 14 | if name not in aggregate_results.keys(): 15 | aggregate_results[name] = [] 16 | try: 17 | float_val = float(value) 18 | except Exception: 19 | float_val = np.nan 20 | aggregate_results[name].append(float_val) 21 | 22 | for name, value in aggregate_results.items(): 23 | metric_name = name 24 | aggregate_results[name] = np.nanmean(value) 25 | if 'pass_rate' in metric_name: 26 | metric_name = metric_name + "(%)" 27 | aggregate_results[name] = aggregate_results[name] * 100.0 28 | aggregate_results[name] = round(aggregate_results[name], 2) 29 | log_metric(metric_name, aggregate_results[name]) 30 | 31 | return aggregate_results 32 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/.promptflow/lkg_sources/concat_scores.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | import numpy as np 6 | import re 7 | 8 | 9 | @tool 10 | def concat_results(groundesness_score: str): 11 | 12 | load_list = [{'name': 'gpt_groundedness', 'score': groundesness_score}] 13 | score_list = [] 14 | errors = [] 15 | for item in load_list: 16 | try: 17 | score = item["score"] 18 | match = re.search(r'\d', score) 19 | if match: 20 | score = match.group() 21 | score = float(score) 22 | except Exception as e: 23 | score = np.nan 24 | errors.append({"name": item["name"], "msg": str(e), "data": item["score"]}) 25 | score_list.append({"name": item["name"], "score": score}) 26 | 27 | variant_level_result = {} 28 | for item in score_list: 29 | item_name = str(item["name"]) 30 | variant_level_result[item_name] = item["score"] 31 | variant_level_result[item_name + '_pass_rate'] = 1 if item["score"] > 3 else 0 32 | return variant_level_result 33 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/.promptflow/lkg_sources/groundedness_score.jinja2: -------------------------------------------------------------------------------- 1 | System: 2 | You are an AI assistant. You will be given the definition of an evaluation metric for assessing the quality of an answer in a question-answering task. Your job is to compute an accurate evaluation score using the provided evaluation metric. 3 | User: 4 | You will be presented with a CONTEXT and an ANSWER about that CONTEXT. You need to decide whether the ANSWER is entailed by the CONTEXT by choosing one of the following rating: 5 | 1. 5: The ANSWER follows logically from the information contained in the CONTEXT. 6 | 2. 1: The ANSWER is logically false from the information contained in the CONTEXT. 7 | 3. an integer score between 1 and 5 and if such integer score does not exists, use 1: It is not possible to determine whether the ANSWER is true or false without further information. 8 | 9 | Read the passage of information thoroughly and select the correct answer from the three answer labels. Read the CONTEXT thoroughly to ensure you know what the CONTEXT entails. 10 | 11 | Note the ANSWER is generated by a computer system, it can contain certain symbols, which should not be a negative factor in the evaluation. 12 | Independent Examples: 13 | ## Example Task #1 Input: 14 | {"CONTEXT": "The Academy Awards, also known as the Oscars are awards for artistic and technical merit for the film industry. They are presented annually by the Academy of Motion Picture Arts and Sciences, in recognition of excellence in cinematic achievements as assessed by the Academy's voting membership. The Academy Awards are regarded by many as the most prestigious, significant awards in the entertainment industry in the United States and worldwide.", "ANSWER": "Oscar is presented every other two years"} 15 | ## Example Task #1 Output: 16 | 1 17 | ## Example Task #2 Input: 18 | {"CONTEXT": "The Academy Awards, also known as the Oscars are awards for artistic and technical merit for the film industry. They are presented annually by the Academy of Motion Picture Arts and Sciences, in recognition of excellence in cinematic achievements as assessed by the Academy's voting membership. The Academy Awards are regarded by many as the most prestigious, significant awards in the entertainment industry in the United States and worldwide.", "ANSWER": "Oscar is very important awards in the entertainment industry in the United States. And it's also significant worldwide"} 19 | ## Example Task #2 Output: 20 | 5 21 | ## Example Task #3 Input: 22 | {"CONTEXT": "In Quebec, an allophone is a resident, usually an immigrant, whose mother tongue or home language is neither French nor English.", "ANSWER": "In Quebec, an allophone is a resident, usually an immigrant, whose mother tongue or home language is not French."} 23 | ## Example Task #3 Output: 24 | 5 25 | ## Example Task #4 Input: 26 | {"CONTEXT": "Some are reported as not having been wanted at all.", "ANSWER": "All are reported as being completely and fully wanted."} 27 | ## Example Task #4 Output: 28 | 1 29 | 30 | Reminder: The return values for each task should be correctly formatted as an integer between 1 and 5. Do not repeat the context. 31 | 32 | ## Actual Task Input: 33 | {"CONTEXT": {{context}}, "ANSWER": {{answer}}} 34 | 35 | Actual Task Output: -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/aggregate_variants_results.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from typing import List 5 | from promptflow import tool, log_metric 6 | import numpy as np 7 | 8 | 9 | @tool 10 | def aggregate_variants_results(results: List[dict]): 11 | aggregate_results = {} 12 | for result in results: 13 | for name, value in result.items(): 14 | if name not in aggregate_results.keys(): 15 | aggregate_results[name] = [] 16 | try: 17 | float_val = float(value) 18 | except Exception: 19 | float_val = np.nan 20 | aggregate_results[name].append(float_val) 21 | 22 | for name, value in aggregate_results.items(): 23 | metric_name = name 24 | aggregate_results[name] = np.nanmean(value) 25 | if 'pass_rate' in metric_name: 26 | metric_name = metric_name + "(%)" 27 | aggregate_results[name] = aggregate_results[name] * 100.0 28 | aggregate_results[name] = round(aggregate_results[name], 2) 29 | log_metric(metric_name, aggregate_results[name]) 30 | 31 | return aggregate_results 32 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/concat_scores.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | import numpy as np 6 | import re 7 | 8 | 9 | @tool 10 | def concat_results(groundesness_score: str): 11 | 12 | load_list = [{'name': 'gpt_groundedness', 'score': groundesness_score}] 13 | score_list = [] 14 | errors = [] 15 | for item in load_list: 16 | try: 17 | score = item["score"] 18 | match = re.search(r'\d', score) 19 | if match: 20 | score = match.group() 21 | score = float(score) 22 | except Exception as e: 23 | score = np.nan 24 | errors.append({"name": item["name"], "msg": str(e), "data": item["score"]}) 25 | score_list.append({"name": item["name"], "score": score}) 26 | 27 | variant_level_result = {} 28 | for item in score_list: 29 | item_name = str(item["name"]) 30 | variant_level_result[item_name] = item["score"] 31 | variant_level_result[item_name + '_pass_rate'] = 1 if item["score"] > 3 else 0 32 | return variant_level_result 33 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/flow.dag.sample.yaml: -------------------------------------------------------------------------------- 1 | inputs: 2 | question: 3 | type: string 4 | default: What feeds all the fixtures in low voltage tracks instead of each light having a line-to-low voltage transformer? 5 | is_chat_input: false 6 | context: 7 | type: string 8 | default: Track lighting, invented by Lightolier, was popular at one period of time because it was much easier to install than recessed lighting, and individual fixtures are decorative and can be easily aimed at a wall. It has regained some popularity recently in low-voltage tracks, which often look nothing like their predecessors because they do not have the safety issues that line-voltage systems have, and are therefore less bulky and more ornamental in themselves. A master transformer feeds all of the fixtures on the track or rod with 12 or 24 volts, instead of each light fixture having its own line-to-low voltage transformer. There are traditional spots and floods, as well as other small hanging fixtures. A modified version of this is cable lighting, where lights are hung from or clipped to bare metal cables under tension 9 | is_chat_input: false 10 | answer: 11 | type: string 12 | default: The main transformer is the object that feeds all the fixtures in low voltage tracks. 13 | is_chat_input: false 14 | outputs: 15 | gpt_groundedness: 16 | type: object 17 | reference: ${concat_scores.output.gpt_groundedness} 18 | evaluation_only: false 19 | is_chat_output: false 20 | nodes: 21 | - name: groundedness_score 22 | type: llm 23 | source: 24 | type: code 25 | path: groundedness_score.jinja2 26 | inputs: 27 | question: ${inputs.question} 28 | context: ${inputs.context} 29 | answer: ${inputs.answer} 30 | max_tokens: 256 31 | deployment_name: mp-aoi-gpt-4-32k 32 | temperature: 0.0 33 | model: gpt-4 34 | api: chat 35 | provider: AzureOpenAI 36 | connection: DRI-Copilot-SQL-OAI 37 | module: promptflow.tools.aoai 38 | aggregation: false 39 | - name: concat_scores 40 | type: python 41 | source: 42 | type: code 43 | path: concat_scores.py 44 | inputs: 45 | groundesness_score: ${groundedness_score.output} 46 | aggregation: false 47 | - name: aggregate_variants_results 48 | type: python 49 | source: 50 | type: code 51 | path: aggregate_variants_results.py 52 | inputs: 53 | results: ${concat_scores.output} 54 | aggregation: true 55 | environment: 56 | python_requirements_txt: requirements.txt 57 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/groundedness_score.jinja2: -------------------------------------------------------------------------------- 1 | System: 2 | You are an AI assistant. You will be given the definition of an evaluation metric for assessing the quality of an answer in a question-answering task. Your job is to compute an accurate evaluation score using the provided evaluation metric. 3 | User: 4 | You will be presented with a CONTEXT and an ANSWER about that CONTEXT. You need to decide whether the ANSWER is entailed by the CONTEXT by choosing one of the following rating: 5 | 1. 5: The ANSWER follows logically from the information contained in the CONTEXT. 6 | 2. 1: The ANSWER is logically false from the information contained in the CONTEXT. 7 | 3. an integer score between 1 and 5 and if such integer score does not exists, use 1: It is not possible to determine whether the ANSWER is true or false without further information. 8 | 9 | Read the passage of information thoroughly and select the correct answer from the three answer labels. Read the CONTEXT thoroughly to ensure you know what the CONTEXT entails. 10 | 11 | Note the ANSWER is generated by a computer system, it can contain certain symbols, which should not be a negative factor in the evaluation. 12 | Independent Examples: 13 | ## Example Task #1 Input: 14 | {"CONTEXT": "The Academy Awards, also known as the Oscars are awards for artistic and technical merit for the film industry. They are presented annually by the Academy of Motion Picture Arts and Sciences, in recognition of excellence in cinematic achievements as assessed by the Academy's voting membership. The Academy Awards are regarded by many as the most prestigious, significant awards in the entertainment industry in the United States and worldwide.", "ANSWER": "Oscar is presented every other two years"} 15 | ## Example Task #1 Output: 16 | 1 17 | ## Example Task #2 Input: 18 | {"CONTEXT": "The Academy Awards, also known as the Oscars are awards for artistic and technical merit for the film industry. They are presented annually by the Academy of Motion Picture Arts and Sciences, in recognition of excellence in cinematic achievements as assessed by the Academy's voting membership. The Academy Awards are regarded by many as the most prestigious, significant awards in the entertainment industry in the United States and worldwide.", "ANSWER": "Oscar is very important awards in the entertainment industry in the United States. And it's also significant worldwide"} 19 | ## Example Task #2 Output: 20 | 5 21 | ## Example Task #3 Input: 22 | {"CONTEXT": "In Quebec, an allophone is a resident, usually an immigrant, whose mother tongue or home language is neither French nor English.", "ANSWER": "In Quebec, an allophone is a resident, usually an immigrant, whose mother tongue or home language is not French."} 23 | ## Example Task #3 Output: 24 | 5 25 | ## Example Task #4 Input: 26 | {"CONTEXT": "Some are reported as not having been wanted at all.", "ANSWER": "All are reported as being completely and fully wanted."} 27 | ## Example Task #4 Output: 28 | 1 29 | 30 | Reminder: The return values for each task should be correctly formatted as an integer between 1 and 5. Do not repeat the context. 31 | 32 | ## Actual Task Input: 33 | {"CONTEXT": {{context}}, "ANSWER": {{answer}}} 34 | 35 | Actual Task Output: -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/evaluation/requirements.txt: -------------------------------------------------------------------------------- 1 | pyodbc -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/.promptflow/flow.detail.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/.promptflow/flow.layout.json: -------------------------------------------------------------------------------- 1 | { 2 | "nodeLayouts": { 3 | "inputs": { 4 | "x": 209, 5 | "y": 62, 6 | "index": -1 7 | }, 8 | "outputs": { 9 | "x": 259, 10 | "y": 502, 11 | "index": -1 12 | }, 13 | "retrieve_customer": { 14 | "x": 614.5, 15 | "y": 172, 16 | "index": 0 17 | }, 18 | "retrieve_past_orders": { 19 | "x": 615, 20 | "y": 282, 21 | "index": 1 22 | }, 23 | "retrieve_products": { 24 | "x": 203.5, 25 | "y": 172, 26 | "index": 2 27 | }, 28 | "retrieve_products_stats": { 29 | "x": 84, 30 | "y": 282, 31 | "index": 3 32 | }, 33 | "chat": { 34 | "x": 434, 35 | "y": 392, 36 | "index": 4 37 | }, 38 | "get_retrieved_documents": { 39 | "x": 84, 40 | "y": 392, 41 | "index": 5 42 | } 43 | }, 44 | "orientation": "Vertical" 45 | } -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/.promptflow/flow.log: -------------------------------------------------------------------------------- 1 | 2023-10-31 16:19:55 -0700 49372 execution.flow INFO Start to run 6 nodes with concurrency level 16. 2 | 2023-10-31 16:19:55 -0700 49372 execution.flow INFO Executing node retrieve_customer. node run id: 716e4432-05cc-4b4c-99c0-1ee568f01b0a_retrieve_customer_0 3 | 2023-10-31 16:19:55 -0700 49372 execution.flow INFO Executing node retrieve_products. node run id: 716e4432-05cc-4b4c-99c0-1ee568f01b0a_retrieve_products_0 4 | 2023-10-31 16:19:57 -0700 49372 execution.flow INFO Node retrieve_customer completes. 5 | 2023-10-31 16:19:57 -0700 49372 execution.flow INFO Executing node retrieve_past_orders. node run id: 716e4432-05cc-4b4c-99c0-1ee568f01b0a_retrieve_past_orders_0 6 | 2023-10-31 16:19:57 -0700 49372 execution.flow INFO Node retrieve_products completes. 7 | 2023-10-31 16:19:57 -0700 49372 execution.flow INFO Executing node retrieve_products_stats. node run id: 716e4432-05cc-4b4c-99c0-1ee568f01b0a_retrieve_products_stats_0 8 | 2023-10-31 16:19:58 -0700 49372 execution.flow INFO Node retrieve_past_orders completes. 9 | 2023-10-31 16:19:58 -0700 49372 execution.flow INFO Node retrieve_products_stats completes. 10 | 2023-10-31 16:19:58 -0700 49372 execution.flow INFO Executing node chat. node run id: 716e4432-05cc-4b4c-99c0-1ee568f01b0a_chat_0 11 | 2023-10-31 16:19:58 -0700 49372 execution.flow INFO Executing node get_retrieved_documents. node run id: 716e4432-05cc-4b4c-99c0-1ee568f01b0a_get_retrieved_documents_0 12 | 2023-10-31 16:19:58 -0700 49372 execution.flow INFO Node get_retrieved_documents completes. 13 | 2023-10-31 16:20:01 -0700 49372 execution.flow WARNING Output of chat is not json serializable, use str to store it. 14 | 2023-10-31 16:20:01 -0700 49372 execution.flow INFO Node chat completes. 15 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/.promptflow/flow.tools.json: -------------------------------------------------------------------------------- 1 | { 2 | "package": {}, 3 | "code": {} 4 | } -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/get_retrieved_documents.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/SQL-AI-samples/35e7a5f365b01d185810f8aa3696c1c509079a07/AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/get_retrieved_documents.cpython-310.pyc -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/retrieve_customer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/SQL-AI-samples/35e7a5f365b01d185810f8aa3696c1c509079a07/AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/retrieve_customer.cpython-310.pyc -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/retrieve_orders.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/SQL-AI-samples/35e7a5f365b01d185810f8aa3696c1c509079a07/AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/retrieve_orders.cpython-310.pyc -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/retrieve_products.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/SQL-AI-samples/35e7a5f365b01d185810f8aa3696c1c509079a07/AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/retrieve_products.cpython-310.pyc -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/retrieve_products_stats.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/SQL-AI-samples/35e7a5f365b01d185810f8aa3696c1c509079a07/AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/__pycache__/retrieve_products_stats.cpython-310.pyc -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/chat.jinja2: -------------------------------------------------------------------------------- 1 | system: 2 | YOU ARE: 3 | You are a helpful assistant that has access to Adenventure sample database which contains tables that stores information of customer, product, product descriptions and order details. 4 | 5 | SITUATION: 6 | You are designed to be able to help answer any questions related to the customer, product recommendations, past order details, products sales information. 7 | 8 | 9 | You are given: 10 | (1) The chat history that you have had with the current user. 11 | (2) The question to answer. 12 | (3) Detailed customer information with their past orders/purchases information retrieved from sql query results. 13 | (3) Detailed products information with description retrieved per this question against a search index. 14 | (4) Aggregated product sales counts information retrieved from sql query results. 15 | 16 | INSTRUCTIONS: 17 | - You MUST find the right information from the retrieved data to answer questions. If no information, please say you don't know. 18 | - Remain grounded, do not invent new facts. 19 | - Since the data is returned by Azure Cognitive Search, be mindful of the importance the search gave to various document. 20 | - Use MARKDOWN to highlight your text. 21 | - Please greeting the users use their name and title that are provided to you. 22 | - Please explain in details and step by step in your answer. 23 | - Make sure to reference any documentation used in the response. 24 | - Reference past orders by name and description that would indicate customer would like the suggested item. It is important to refer customer information and past orders. 25 | 26 | YOUR TASK: 27 | Your task is, based on the question of the user (and to a lower extent the chat history), to find the right information from the retrieved data. That data consists of (1) structured in JSON as returned by Azure Cognitive Search indexes. (2) Output json from sql query directly. 28 | 29 | system: 30 | The following are the information that you used to answer customer' question. 31 | 32 | #Products Information: 33 | You are given the following products as reference to your response. 34 | {% for item in retrieved_products %} 35 | ProductCategory: {{item.ProductCategoryName}} 36 | ProductName: {{item.Name}} 37 | ProductColor: {{item.Color}} 38 | ProductListPrice: {{item.ListPrice}} 39 | ProductDescription: {{item.Description}} 40 | {% endfor %} 41 | 42 | #Aggregated Products Sales Count: 43 | You are given the following products sales summary which is ordered by product category name and sales count. 44 | {% for item in retrieved_products_stats%} 45 | ProductCategory: {{item.ProductCategoryName}} 46 | ProductName: {{item.Name}} 47 | ProductSalesCountTotal: {{item.sales_count}} 48 | ProductListPrice: {{item.ListPrice}} 49 | ProductColor:{{item.Color}} 50 | {% endfor %} 51 | 52 | # Previous Purchases: 53 | Here is the details of the user's past purchases, use it as additional context to the question they are asking and as relevant information pertaining to their question. 54 | {% for item in retrieved_orders %} 55 | ProductName: {{item.Name}} 56 | ProductColor: {{item.Color}} 57 | ProductDescription: {{item.Description}} 58 | ProductListPrice: {{item.ListPrice}} 59 | ProductSize: {{item.Size}} 60 | {% endfor %} 61 | 62 | # Customer Context: 63 | {% for item in retrieved_customers %} 64 | The customer's name is {{item.FirstName}}, last name is {{item.LastName}} and title is {{item.Title}}. 65 | {% endfor %} 66 | 67 | Chat history: 68 | {% for item in chat_history %} 69 | user: 70 | {{item.inputs.question}} 71 | assistant: 72 | {{item.outputs.answer}} 73 | {% endfor %} 74 | 75 | Additional INSTRUCTIONS: 76 | If customer is asking for recommendations, then reference on products specifically by name, size, color, price and description that would indicate the customer might like the suggested product. 77 | 78 | If customer is asking for recommendations, refer to their past purchased products to show that they might like the recommended products. 79 | 80 | If customer is aksing for general products information, then please answer the question by summarizing the data. Please use step by step reasoning and provide as many details as you can. DONOT provide irrelevant information. 81 | 82 | user: 83 | {{question}} -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/get_retrieved_documents.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | 6 | # The inputs section will change based on the arguments of the tool function, after you save the code 7 | # Adding type to arguments and return value will help the system show the types properly 8 | # Please update the function name/signature per need 9 | @tool 10 | def my_python_tool(input1: list, input2: list, input3: list, input4: list) -> str: 11 | retrieved_documents = input1 + input2 + input3 + input4 12 | return retrieved_documents -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/requirements.txt: -------------------------------------------------------------------------------- 1 | pyodbc -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/retrieve_customer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | import pandas as pd 6 | from promptflow.connections import CustomConnection 7 | import pyodbc 8 | import json 9 | 10 | @tool 11 | def get_customer_details(inputs: dict, conn: CustomConnection): 12 | # this is a bug in promptflow where they treat this input type differently 13 | if type(inputs) == str: 14 | inputs_dict = eval(inputs) 15 | else: 16 | inputs_dict = inputs 17 | if inputs_dict['MiddleName'] == "": 18 | sqlQuery = f"""select * from [SalesLT].[Customer] WHERE FirstName='{inputs_dict['FirstName']}' and MiddleName is NULL and LastName='{inputs_dict['LastName']}'""" 19 | else: 20 | sqlQuery = f"""select * from [SalesLT].[Customer] WHERE FirstName='{inputs_dict['FirstName']}' and MiddleName='{inputs_dict['MiddleName']}' and LastName='{inputs_dict['LastName']}'""" 21 | connectionString = conn['connectionString'] 22 | sqlConn = pyodbc.connect(connectionString) 23 | cursor = sqlConn.cursor() 24 | queryResult = pd.DataFrame() 25 | try: 26 | cursor.execute(sqlQuery) 27 | records = cursor.fetchall() 28 | queryResult = pd.DataFrame.from_records(records, columns=[col[0] for col in cursor.description]) 29 | except Exception as e: 30 | print(f"connection could not be established: {e}") 31 | finally: 32 | cursor.close() 33 | 34 | customer_detail_json = json.loads(queryResult.to_json(orient='records')) 35 | return customer_detail_json -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/retrieve_orders.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | import pandas as pd 6 | from promptflow.connections import CustomConnection 7 | import pyodbc 8 | import json 9 | import re 10 | 11 | 12 | def get_customer_id(inputs: list): 13 | customer_ids_list = set() 14 | for i in range(len(inputs)): 15 | customer_id = inputs[i]['CustomerID'] 16 | if customer_id not in customer_ids_list: 17 | customer_ids_list.add(customer_id) 18 | return re.sub(r',(?=\))', '', str(tuple(customer_ids_list))) 19 | 20 | 21 | @tool 22 | def get_customer_past_orders(inputs: list, conn: CustomConnection): 23 | customers_ids = get_customer_id(inputs) 24 | sqlQuery = """select SOH.CustomerID, SOD.ProductID, SP.Name, SP.ProductNumber, SP.Color, SP.Size, SP.ListPrice, SP.ProductCategoryID, SP.ProductModelID, PD.ProductDescriptionID, PD.Description 25 | from [SalesLT].[SalesOrderDetail] SOD 26 | INNER JOIN [SalesLT].[SalesOrderHeader] SOH on SOD.SalesOrderID = SOH.SalesOrderID 27 | INNER JOIN [SalesLT].[Product] SP ON SP.ProductID = SOD.ProductID 28 | INNER JOIN [SalesLT].[ProductModelProductDescription] PMPD ON PMPD.ProductModelID = SP.ProductModelID 29 | INNER JOIN [SalesLT].[ProductDescription] PD ON PD.ProductDescriptionID = PMPD.ProductDescriptionID 30 | WHERE PMPD.Culture = 'en' 31 | AND CustomerID IN {customers_ids}""".replace("{customers_ids}", customers_ids) 32 | connectionString = conn['connectionString'] 33 | conn = pyodbc.connect(connectionString) 34 | cursor = conn.cursor() 35 | queryResult = pd.DataFrame() 36 | try: 37 | cursor.execute(sqlQuery) 38 | records = cursor.fetchall() 39 | queryResult = pd.DataFrame.from_records( 40 | records, columns=[col[0] for col in cursor.description]) 41 | except Exception as e: 42 | print(f"connection could not be established: {e}") 43 | finally: 44 | cursor.close() 45 | 46 | return json.loads(queryResult.to_json(orient='records')) 47 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/retrieve_products.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | import requests 6 | import json 7 | import os 8 | import openai 9 | import re 10 | from promptflow.connections import CustomConnection, AzureOpenAIConnection 11 | from azure.core.credentials import AzureKeyCredential 12 | 13 | 14 | def generate_embeddings(text, conn: CustomConnection): 15 | openai.api_base = conn['OPENAI_API_BASE_EMBED'] 16 | openai.api_type = "azure" 17 | # this may change in the future 18 | openai.api_version = conn['OPENAI_API_VERSION'] 19 | openai.api_key = conn['OPENAI_API_KEY_EMBED'] 20 | response = openai.Embedding.create( 21 | input=text, engine="text-embedding-ada-002") 22 | embeddings = response['data'][0]['embedding'] 23 | return embeddings 24 | 25 | # The inputs section will change based on the arguments of the tool function, after you save the code 26 | # Adding type to arguments and return value will help the system show the types properly 27 | # Please update the function name/signature per need 28 | 29 | 30 | @tool 31 | def get_products(search_text: str, conn: CustomConnection, top_k: int) -> str: 32 | search_service = "sqldricopilot" 33 | index_name = "promptflow-demo-product-description" 34 | search_key = conn["search-key"] 35 | api_version = "2023-07-01-Preview" 36 | 37 | headers = { 38 | 'Content-Type': 'application/json', 39 | 'api-key': search_key, 40 | } 41 | params = { 42 | 'api-version': api_version, 43 | } 44 | body = { 45 | "vector": { 46 | "value": generate_embeddings(text=search_text, conn=conn), 47 | "fields": "DescriptionVector, ProductCategoryNameVector", 48 | "k": top_k 49 | }, 50 | "search": search_text, 51 | "select": "ProductId, ProductCategoryName, Name, ProductNumber, Color, ListPrice, Size, ProductCategoryID, ProductModelID, ProductDescriptionID, Description", 52 | "top": top_k, 53 | } 54 | response = requests.post( 55 | f"https://{search_service}.search.windows.net/indexes/{index_name}/docs/search", headers=headers, params=params, json=body) 56 | response_json = response.json()['value'] 57 | for i in range(len(response_json)): 58 | response_json[i]['ProductId'] = int(response_json[i]['ProductId']) 59 | 60 | return response_json 61 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/retrieve_products_stats.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | import pandas as pd 6 | from promptflow.connections import CustomConnection 7 | import pyodbc 8 | import json 9 | import re 10 | 11 | 12 | def get_product_category_name(inputs: list): 13 | product_category_name_lists = set() 14 | for i in range(len(inputs)): 15 | pc_name = inputs[i]['ProductCategoryName'] 16 | if pc_name not in product_category_name_lists: 17 | product_category_name_lists.add(pc_name) 18 | return re.sub(r',(?=\))', '', str(tuple(product_category_name_lists))) 19 | 20 | 21 | @tool 22 | def get_product_stats(inputs: list, conn: CustomConnection): 23 | product_category_name_lists = get_product_category_name(inputs) 24 | sqlQuery = """SELECT TOP 10 pc.Name AS ProductCategoryName, p.Name, p.Color, p.ListPrice, count(p.Name) as sales_count 25 | FROM SalesLT.Product p 26 | JOIN SalesLT.SalesOrderDetail sod 27 | ON p.ProductID = sod.ProductID 28 | JOIN SalesLT.ProductCategory pc 29 | ON pc.ProductCategoryID = p.ProductCategoryID 30 | WHERE pc.Name IN {product_category_name_lists} 31 | GROUP BY pc.Name, p.Name, p.Color, p.ListPrice 32 | ORDER by pc.Name, sales_count DESC""".replace("{product_category_name_lists}", product_category_name_lists) 33 | connectionString = conn['connectionString'] 34 | conn = pyodbc.connect(connectionString) 35 | cursor = conn.cursor() 36 | queryResult = pd.DataFrame() 37 | try: 38 | cursor.execute(sqlQuery) 39 | records = cursor.fetchall() 40 | queryResult = pd.DataFrame.from_records( 41 | records, columns=[col[0] for col in cursor.description]) 42 | except Exception as e: 43 | print(f"connection could not be established: {e}") 44 | finally: 45 | cursor.close() 46 | 47 | return json.loads(queryResult.to_json(orient='records')) 48 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow/single_run_data.jsonl: -------------------------------------------------------------------------------- 1 | {"chat_history": [], "question": "I'm looking for new bike jerseys for my customers", "customer": {"FirstName":"Donald","MiddleName":"L.","LastName":"Blanton"}} -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/chat.jinja2: -------------------------------------------------------------------------------- 1 | system: 2 | YOU ARE: 3 | You are a helpful assistant that has access to Adenventure sample database which contains tables that stores information of user, product, product descriptions and order details. 4 | 5 | SITUATION: 6 | You are designed to be able to help answer any questions related to the customer, product recommendations, past order details, products sales information. 7 | 8 | You are given: 9 | (1) The chat history that you have had with the current user. 10 | (2) The question to answer. 11 | (3) Detailed product information with description retrieved per this question against a search index. 12 | (4) User's information including name and title. 13 | (5) User's past order details retrieved from sql query. 14 | 15 | INSTRUCTIONS: 16 | - You MUST find the right information from the retrieved data to answer questions. If no relevant information provided, please say you don't know, DO NOT invent new facts. For example, if there is nothing showed in #Previous purchases#, just tell the user that they have no purchases in the past. 17 | - Remain grounded, do not invent new facts. 18 | - Since the data is returned by Azure Cognitive Search, be mindful of the importance the search gave to various document. 19 | - Use MARKDOWN to highlight your text. 20 | - You MUST greet the user using the name and title that are provided to you, for example, say "Hello Mr. Liu". 21 | - Please explain in details and step by step in your answer. 22 | - Make sure to reference any documentation used in the response. 23 | - Reference past orders by name and relevant information like color, size, and description that would indicate user would like the suggested item. It is important to refer user information and past orders. 24 | - When giving recommendation, you MUST recommend features including color, size that the user haven't purchased before based on their purchase history. For example, if the user have purchased a product of black color and size Medium, you can say that now we also have this product with yellow color and Large size in the inventory that the user may want to try. 25 | - When describing products, make sure to refer to its color, size, price, description, and other useful information. 26 | - DO NOT create new product with new features, answer question based on documents provided to you. 27 | 28 | 29 | YOUR TASK: 30 | Your task is, based on the question of the user (and to a lower extent the chat history), to find the right information from the retrieved data. That data consists of (1) structured in JSON as returned by Azure Cognitive Search indexes. (2) Output json from sql query directly. 31 | 32 | system: 33 | # Products: 34 | You are given the following products as reference to your response. 35 | {% for item in retrieved_products %} 36 | {{item}} 37 | {% endfor %} 38 | 39 | # Product sales summary 40 | You are given some most saled (most popular) products for some given product categories, please use it as reference if the user is asking about recommendation about these categories. 41 | {% for item in retrieved_sales_stat %} 42 | {{item}} 43 | {% endfor %} 44 | 45 | # User context: 46 | The user's first name is {{retrieved_customers[0].FirstName}}, last name is {{retrieved_customers[0].LastName}}, title is {{retrieved_customers[0].Title}}. 47 | 48 | # Previous purchases: 49 | Here is the user's past purchases, use it as additional context to what the user is asking. 50 | {% for item in retrieved_orders %} 51 | {{item}} 52 | {% endfor %} 53 | 54 | # Chat history: 55 | {% for item in chat_history %} 56 | user: 57 | {{item.inputs.question}} 58 | assistant: 59 | {{item.outputs.answer}} 60 | {% endfor %} 61 | 62 | user: 63 | {{question}} -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/flow.dag.sample.yaml: -------------------------------------------------------------------------------- 1 | id: template_chat_flow 2 | name: Template Chat Flow 3 | inputs: 4 | chat_history: 5 | type: list 6 | default: [] 7 | is_chat_input: false 8 | is_chat_history: true 9 | question: 10 | type: string 11 | default: Hello 12 | is_chat_input: true 13 | customer: 14 | type: string 15 | default: Donald Blanton 16 | is_chat_input: false 17 | outputs: 18 | answer: 19 | type: string 20 | reference: ${chat.output} 21 | is_chat_output: true 22 | retrieved_documents: 23 | type: string 24 | reference: ${get_retrieved_documents.output} 25 | nodes: 26 | - name: sql_query_store 27 | type: python 28 | source: 29 | type: code 30 | path: sql_query_store.py 31 | inputs: {} 32 | use_variants: false 33 | - name: get_customer 34 | type: python 35 | source: 36 | type: code 37 | path: get_customer.py 38 | inputs: 39 | conn_db: dummy 40 | customer: ${inputs.customer} 41 | use_variants: false 42 | - name: get_past_orders 43 | type: python 44 | source: 45 | type: code 46 | path: get_pastorders.py 47 | inputs: 48 | conn_db: dummy 49 | customer: ${get_customer.output} 50 | sql_query_prep: ${sql_query_store.output} 51 | use_variants: false 52 | - name: get_product 53 | type: python 54 | source: 55 | type: code 56 | path: get_product.py 57 | inputs: 58 | conn_db: dummy 59 | conn: dummy 60 | search_text: ${inputs.question} 61 | sql_query_prep: ${sql_query_store.output} 62 | top_k: 5 63 | use_variants: false 64 | - name: get_sales_stat 65 | type: python 66 | source: 67 | type: code 68 | path: get_produt_stats.py 69 | inputs: 70 | conn_db: dummy 71 | products: ${get_product.output} 72 | sql_query_prep: ${sql_query_store.output} 73 | use_variants: false 74 | - name: get_retrieved_documents 75 | type: python 76 | source: 77 | type: code 78 | path: get_retrieved_documents.py 79 | inputs: 80 | input1: ${get_past_orders.output} 81 | input2: ${get_product.output} 82 | input3: ${get_sales_stat.output} 83 | use_variants: false 84 | - name: chat 85 | type: llm 86 | source: 87 | type: code 88 | path: chat.jinja2 89 | inputs: 90 | deployment_name: dummy 91 | temperature: 0 92 | top_p: 1 93 | stop: "" 94 | max_tokens: 1000 95 | presence_penalty: 0 96 | frequency_penalty: 0 97 | logit_bias: "" 98 | chat_history: ${inputs.chat_history} 99 | question: ${inputs.question} 100 | retrieved_customers: ${get_customer.output} 101 | retrieved_orders: ${get_past_orders.output} 102 | retrieved_products: ${get_product.output} 103 | retrieved_sales_stat: ${get_sales_stat.output} 104 | provider: AzureOpenAI 105 | connection: dummy 106 | api: chat 107 | module: promptflow.tools.aoai 108 | use_variants: false 109 | node_variants: {} 110 | environment: 111 | python_requirements_txt: requirements.txt 112 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/get_customer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | from promptflow.connections import CustomConnection 6 | 7 | import pyodbc 8 | import pandas as pd 9 | import numpy as np 10 | import sqlalchemy as sa 11 | import json 12 | 13 | def execute_sql(sql_query: str, conn_db: CustomConnection): 14 | 15 | conn_string = conn_db['connection-string'] 16 | with pyodbc.connect(conn_string,autocommit=True) as conn: 17 | with conn.cursor() as cursor: 18 | cursor.execute(sql_query) 19 | query_out = cursor.fetchall() 20 | 21 | toReturn = pd.DataFrame((tuple(t) for t in query_out)) 22 | toReturn.columns = [column[0] for column in cursor.description] 23 | 24 | return toReturn 25 | 26 | 27 | @tool 28 | def get_customer(customer: str, conn_db: CustomConnection): 29 | first_name = customer.split()[0] 30 | last_name = customer.split()[-1] 31 | customer_query = f"""select * from [SalesLT].[Customer] 32 | WHERE FirstName='{first_name}' AND LastName='{last_name}'""" 33 | 34 | out_df = execute_sql(sql_query=customer_query, conn_db=conn_db) 35 | out_json = out_df.to_json(orient="records") 36 | out_dict = json.loads(out_json) 37 | 38 | return out_dict -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/get_pastorders.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | from promptflow.connections import CustomConnection 6 | 7 | import pyodbc 8 | import pandas as pd 9 | import numpy as np 10 | import sqlalchemy as sa 11 | import json 12 | 13 | 14 | def execute_sql(sql_query: str, conn_db: CustomConnection): 15 | 16 | conn_string = conn_db['connection-string'] 17 | with pyodbc.connect(conn_string,autocommit=True) as conn: 18 | with conn.cursor() as cursor: 19 | cursor.execute(sql_query) 20 | query_out = cursor.fetchall() 21 | 22 | toReturn = pd.DataFrame((tuple(t) for t in query_out)) 23 | toReturn.columns = [column[0] for column in cursor.description] 24 | 25 | return toReturn 26 | 27 | 28 | @tool 29 | def get_orders(customer: list, sql_query_prep: dict, conn_db:CustomConnection): 30 | 31 | list_cust_id = list(map(lambda x: x['CustomerID'], customer)) 32 | list_cust_id = str(tuple(list_cust_id)).replace(",)", ")") 33 | order_query = sql_query_prep['query_order'].replace("{list_cust}", list_cust_id) 34 | 35 | try: 36 | out_df = execute_sql(sql_query=order_query, conn_db=conn_db) 37 | out_json = out_df.to_json(orient="records") 38 | out_dict = json.loads(out_json) 39 | except: 40 | out_dict = {} 41 | 42 | return out_dict -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/get_product.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | from promptflow.connections import CustomConnection 6 | from azure.core.credentials import AzureKeyCredential 7 | import requests 8 | import json 9 | import os 10 | import openai 11 | import re 12 | import pyodbc 13 | import pandas as pd 14 | import numpy as np 15 | import sqlalchemy as sa 16 | 17 | 18 | def generate_embeddings(text, conn: CustomConnection): 19 | openai.api_base = conn.OPENAI_API_BASE_EMBED 20 | openai.api_key = conn.OPENAI_API_KEY_EMBED 21 | openai.api_version = conn.OPENAI_API_VERSION 22 | openai.api_type = "azure" 23 | 24 | response = openai.Embedding.create( 25 | input=text, engine="text-embedding-ada-002") 26 | embeddings = response['data'][0]['embedding'] 27 | return embeddings 28 | 29 | def execute_sql(sql_query: str, conn_db: CustomConnection): 30 | 31 | conn_string = conn_db['connection-string'] 32 | with pyodbc.connect(conn_string,autocommit=True) as conn: 33 | with conn.cursor() as cursor: 34 | cursor.execute(sql_query) 35 | query_out = cursor.fetchall() 36 | 37 | toReturn = pd.DataFrame((tuple(t) for t in query_out)) 38 | toReturn.columns = [column[0] for column in cursor.description] 39 | 40 | return toReturn 41 | 42 | @tool 43 | def get_product(search_text: str, sql_query_prep: dict, conn: CustomConnection, conn_db: CustomConnection, top_k:int) -> str: 44 | search_service = "sqldricopilot" 45 | index_name = "promptflow-demo-product-description" 46 | search_key = conn["acs-search-key"] 47 | api_version = "2023-07-01-Preview" 48 | 49 | headers = { 50 | 'Content-Type': 'application/json', 51 | 'api-key': search_key, 52 | } 53 | params = { 54 | 'api-version': api_version, 55 | } 56 | body = { 57 | "vector": { 58 | "value": generate_embeddings(text = search_text, conn = conn), 59 | "fields": "DescriptionVector, ProductCategoryNameVector", 60 | "k": top_k 61 | }, 62 | "search": search_text, 63 | "select": "ProductId, ProductCategoryName, Name, ProductNumber, Color, ListPrice, Size, ProductCategoryID, ProductModelID, ProductDescriptionID, Description", 64 | "top": top_k, 65 | } 66 | response = requests.post( 67 | f"https://{search_service}.search.windows.net/indexes/{index_name}/docs/search", headers=headers, params=params, json=body) 68 | response_json = response.json()['value'] 69 | 70 | list_prod_id = list(map(lambda x: x['ProductId'], response_json)) 71 | list_prod_id = str(tuple(list_prod_id)).replace(",)", ")") 72 | query_product = sql_query_prep['query_prod_byID'].replace("{list_product}", list_prod_id) 73 | 74 | try: 75 | out_df = execute_sql(sql_query=query_product, conn_db=conn_db) 76 | out_json = out_df.to_json(orient="records") 77 | out_dict = json.loads(out_json) 78 | except: 79 | out_dict = {} 80 | 81 | return out_dict -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/get_produt_stats.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | from promptflow.connections import CustomConnection 6 | from azure.core.credentials import AzureKeyCredential 7 | import requests 8 | import json 9 | import os 10 | import openai 11 | import re 12 | import pyodbc 13 | import pandas as pd 14 | import numpy as np 15 | import sqlalchemy as sa 16 | 17 | def execute_sql(sql_query: str, conn_db: CustomConnection): 18 | 19 | conn_string = conn_db['connection-string'] 20 | with pyodbc.connect(conn_string,autocommit=True) as conn: 21 | with conn.cursor() as cursor: 22 | cursor.execute(sql_query) 23 | query_out = cursor.fetchall() 24 | 25 | toReturn = pd.DataFrame((tuple(t) for t in query_out)) 26 | toReturn.columns = [column[0] for column in cursor.description] 27 | 28 | return toReturn 29 | 30 | @tool 31 | def get_sales_stat(products: list, sql_query_prep: dict, conn_db: CustomConnection): 32 | 33 | list_cate_id = list(map(lambda x: x['ProductCategoryID'], products)) 34 | list_cate_id = str(tuple(list_cate_id)).replace(",)", ")") 35 | query_sales_stat = sql_query_prep['query_sales_stat'].replace("{list_cate}", list_cate_id) 36 | 37 | try: 38 | out_df = execute_sql(sql_query=query_sales_stat, conn_db=conn_db) 39 | out_json = out_df.to_json(orient="records") 40 | out_dict = json.loads(out_json) 41 | except: 42 | out_dict = {} 43 | 44 | return out_dict -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/get_retrieved_documents.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | 6 | # The inputs section will change based on the arguments of the tool function, after you save the code 7 | # Adding type to arguments and return value will help the system show the types properly 8 | # Please update the function name/signature per need 9 | @tool 10 | def my_python_tool(input1: list, input2: list, input3: list) -> str: 11 | retrieved_documents = { 12 | "User past orders": input1, 13 | "Retrieved relevant products": input2, 14 | "Retrieved product sales stats": input3, 15 | } 16 | return retrieved_documents -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/requirements.txt: -------------------------------------------------------------------------------- 1 | pyodbc -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/single_run_data.jsonl: -------------------------------------------------------------------------------- 1 | {"chat_history": [], "question": "I'm looking for new bike jerseys for my customers", "customer": "Donald Blanton"} -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/promptflow_v2/sql_query_store.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Microsoft Corporation. 2 | # Licensed under the MIT license. 3 | 4 | from promptflow import tool 5 | 6 | # English only product description 7 | query_prod_detail = """ 8 | WITH pro_desc AS( 9 | SELECT P1.ProductDescriptionID, P1.Description, P2.ProductModelID, 10 | ROW_NUMBER() OVER(PARTITION BY P2.ProductModelID ORDER BY P1.ProductDescriptionID) AS row_number 11 | FROM SalesLT.ProductDescription AS P1 12 | INNER JOIN SalesLT.ProductModelProductDescription AS P2 13 | ON P2.ProductDescriptionID = P1.ProductDescriptionID 14 | ), prod_desc_eng AS( 15 | -- English only product descriptions 16 | SELECT * FROM pro_desc 17 | WHERE row_number=1 18 | ), prod_detail AS( 19 | SELECT p.ProductID, p.ProductModelID, p.Name, p.Color, p.Size, p.Weight, p.ListPrice, p_des.Description, p_cate.Name AS Category, p_cate.ProductCategoryID 20 | FROM SalesLT.Product AS p 21 | INNER JOIN prod_desc_eng as p_des 22 | ON p.ProductModelID = p_des.ProductModelID 23 | INNER JOIN SalesLT.ProductCategory AS p_cate 24 | ON p_cate.ProductCategoryID = p.ProductCategoryID 25 | ) 26 | """ 27 | 28 | # Customer order history 29 | query_order = query_prod_detail + """ 30 | SELECT p.Name, p.Category, p.Color, p.Size, p.Weight, p.ListPrice, p.Description 31 | FROM prod_detail AS p 32 | INNER JOIN 33 | SalesLT.SalesOrderDetail AS sod 34 | ON sod.ProductID = p.ProductID 35 | INNER JOIN SalesLT.SalesOrderHeader AS soh 36 | ON sod.SalesOrderID = soh.SalesOrderID 37 | WHERE soh.CustomerID IN {list_cust}""" 38 | 39 | # product detail by id 40 | query_prod_byID = query_prod_detail + """ 41 | SELECT p.Name, p.Category, p.Color, p.Size, p.Weight, p.ListPrice, p.Description, p.ProductCategoryID 42 | FROM prod_detail AS p 43 | WHERE p.ProductID IN {list_product}""" 44 | 45 | # product sales stats by category id, returns top 5 most saled products for each category in the list 46 | query_sales_stat = query_prod_detail + """, prod_sales AS( 47 | SELECT p.Name, p.Category, p.Color, p.Size, p.Weight, p.ListPrice, p.Description, count(p.Name) as sales_count, p.ProductCategoryID, 48 | ROW_NUMBER() OVER(PARTITION BY p.ProductCategoryID ORDER BY count(p.Name) DESC) AS row_number 49 | FROM prod_detail p 50 | INNER JOIN SalesLT.SalesOrderDetail sod 51 | ON p.ProductID = sod.ProductID 52 | INNER JOIN SalesLT.ProductCategory pc 53 | ON pc.ProductCategoryID = p.ProductCategoryID 54 | WHERE pc.ProductCategoryID IN {list_cate} 55 | GROUP BY p.Name, p.Category, p.Color, p.Size, p.Weight, p.ListPrice, p.Description, p.ProductCategoryID 56 | ) 57 | SELECT p.Name, p.Category, p.Color, p.Size, p.Weight, p.ListPrice, p.Description, p.sales_count 58 | FROM prod_sales AS p 59 | WHERE p.row_number <= 5""" 60 | 61 | @tool 62 | def sql_query_prep(): 63 | 64 | return { 65 | 'query_order': query_order, 66 | 'query_prod_byID': query_prod_byID, 67 | 'query_sales_stat': query_sales_stat 68 | } -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/run.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run the PromptFlow locally. 3 | 4 | Copyright (c) Microsoft Corporation. 5 | Licensed under the MIT license. 6 | """ 7 | 8 | # %% 9 | from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential 10 | # azure version promptflow apis 11 | from promptflow.azure import PFClient 12 | from promptflow.entities import AzureOpenAIConnection, CustomConnection 13 | from promptflow.entities import Run 14 | import json 15 | import os 16 | from azure.core.credentials import TokenCredential 17 | 18 | print("Loading configs from file.") 19 | with open(f'configs/flow_config.json') as f: 20 | config = json.load(f) 21 | 22 | # Getting credentials for local access 23 | # ----------------------------------------------------------------------------- 24 | 25 | # %% 26 | try: 27 | credential: TokenCredential = DefaultAzureCredential() 28 | # Check if given credential can get token successfully. 29 | credential.get_token("https://management.azure.com/.default") 30 | except Exception: 31 | # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work 32 | credential = InteractiveBrowserCredential() 33 | # %% 34 | # Get a handle to workspace 35 | pf = PFClient( 36 | credential=credential, 37 | # this will look like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 38 | subscription_id=config["subscription_id"], 39 | resource_group_name=config["resource_group_name"], 40 | workspace_name=config["workspace_name"], 41 | ) 42 | 43 | print("currently Azure Promptflow SDK don't support create connection and upload to workspace, so we need to create connection manually IN PORTAL") 44 | # Loading PromptFLow 45 | # %% 46 | # load flow 47 | flow_path = "./promptflow" 48 | data_path = "./data/batch_run_data.jsonl" 49 | # assume you have existing runtime with this name provisioned 50 | runtime = config["promptflow_runtime"] 51 | 52 | # %% 53 | # create run 54 | base_run = pf.run( 55 | flow=flow_path, 56 | data=data_path, 57 | runtime=runtime, 58 | column_mapping={ # map the url field from the data to the url input of the flow 59 | "chat_history": "${data.chat_history}", 60 | "question": "${data.question}", 61 | "customer": "${data.customer}" 62 | }, 63 | display_name="sql-promptflow-demo-1" 64 | ) 65 | pf.stream(base_run) 66 | # %% 67 | details = pf.get_details(base_run) 68 | details.head(10) 69 | 70 | pf.visualize(base_run) 71 | # %% 72 | -------------------------------------------------------------------------------- /AzureSQLPromptFlowSamples/src/sql-promptflow-demo/setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file do the setup for testing the flow locally. 3 | 4 | Copyright (c) Microsoft Corporation. 5 | Licensed under the MIT license. 6 | """ 7 | # %% 8 | from promptflow import PFClient 9 | from promptflow.entities import AzureOpenAIConnection, CustomConnection 10 | import json 11 | import os 12 | import yaml 13 | import subprocess 14 | 15 | from azure.identity import DefaultAzureCredential 16 | from azure.keyvault.secrets import SecretClient 17 | 18 | pf = PFClient() 19 | # %% 20 | # Keyvault Setup 21 | 22 | 23 | def get_keyvault_secret(keyvault_uri: str, secret_name: str): 24 | """Use the default credential (e.g., az login) to get key vault access and retrieve a secret.""" 25 | credential = DefaultAzureCredential( 26 | exclude_shared_token_cache_credential=True, exclude_visual_studio_credential=True) 27 | client = SecretClient(vault_url=keyvault_uri, credential=credential) 28 | secret = client.get_secret(secret_name) 29 | return secret.value 30 | 31 | 32 | def upload_secret(vault_url, key, value): 33 | """Use the default credential (e.g., az login) to get key vault access and set a secret.""" 34 | 35 | # changing key to lowercase and replacing underscores with dashes 36 | key = key.lower().replace('_', '-') 37 | 38 | # Use the default credential (e.g., az login) 39 | credential = DefaultAzureCredential( 40 | exclude_shared_token_cache_credential=True, exclude_visual_studio_credential=True) 41 | client = SecretClient(vault_url=vault_url, credential=credential) 42 | 43 | # Set (or update) the secret 44 | client.set_secret(key, value) 45 | 46 | 47 | # Load the config json files 48 | print("Loading flow config files...") 49 | with open('./configs/flow_config.json') as f: 50 | config = json.load(f) 51 | 52 | print("Loading keys from environment variables...") 53 | with open('./configs/key_config_local.json') as f: 54 | keys = json.load(f) 55 | 56 | 57 | # Send config_local.json contents to key vault 58 | print('Uploading all secrets from key_config_local.json to Key Vault') 59 | print('Note: keys will be converted to lowercase and underscores will be replaced with dashes (Key Vault requirement)') 60 | for key, value in keys.items(): 61 | print(f'Uploading secret {key}') 62 | upload_secret(config['keyvault_uri'], key, value) 63 | print(f'Secret {key} uploaded successfully') 64 | 65 | # %% 66 | # setting up AOAI connection 67 | print("Setting up AOAI connections.") 68 | print('Getting AOAI API key from keyvault') 69 | aoai_api_key = get_keyvault_secret(config['keyvault_uri'], 'aoai-api-key') 70 | 71 | connection = AzureOpenAIConnection( 72 | name=config['azure_open_ai_connection_name'], 73 | api_key=aoai_api_key, 74 | api_base=config["aoai_api_base"], 75 | api_type=config["aoai_api_type"], 76 | api_version=config["aoai_api_version"], 77 | ) 78 | conn = pf.connections.create_or_update(connection) 79 | print("successfully created connection") 80 | # %% 81 | # setting up SQL connection 82 | print("Setting up SQL connections.") 83 | print('Getting SQL Connection STRING from keyvault') 84 | connection_string = get_keyvault_secret(config['keyvault_uri'], 'connection-string') 85 | 86 | connection = CustomConnection( 87 | name=config['SQLDB_connection_name'], 88 | secrets={"connection-string": connection_string} 89 | ) 90 | conn = pf.connections.create_or_update(connection) 91 | print("successfully created connection") 92 | 93 | # setting up ACS/Embedding connection 94 | print("Setting up ACS connections.") 95 | print('Getting ACS/Embedding connection STRING from keyvault') 96 | acs_key = get_keyvault_secret(config['keyvault_uri'], 'acs-key') 97 | aoai_api_key_embed = get_keyvault_secret(config['keyvault_uri'], 'aoai-api-key-embed') 98 | 99 | connection = CustomConnection( 100 | name=config['ACS_connection_name'], 101 | secrets={"acs-search-key": acs_key, 102 | "OPENAI_API_BASE_EMBED": config["OPENAI_API_BASE_EMBED"], "OPENAI_API_KEY_EMBED": aoai_api_key_embed, 103 | "OPENAI_API_VERSION": config["OPENAI_API_VERSION"]} 104 | ) 105 | # Create the connection, note that all secret values will be scrubbed in the returned result 106 | conn = pf.connections.create_or_update(connection) 107 | print("successfully created connection") 108 | 109 | # %% 110 | # Load the yaml file to dictionary path is azure_openai.yml 111 | print("Setting up flow.dag.yaml.") 112 | with open('./promptflow_v2/flow.dag.sample.yaml') as f: 113 | config_flow = yaml.load(f, Loader=yaml.FullLoader) 114 | # import pdb; pdb.set_trace() 115 | # replace the deployment_name with the one you want to use 116 | for node in config_flow["nodes"]: 117 | # Setting up model for the agent chat node. 118 | if node.get('api') == "chat": 119 | if 'deployment_name' in node: 120 | node["deployment_name"] = config['aoai_deployment_name'] 121 | if 'deployment_name' in node['inputs']: 122 | node['inputs']['deployment_name'] = config['aoai_deployment_name'] 123 | # Setting up model for the final chat node. 124 | if 'connection' in node: 125 | node['connection'] = config['azure_open_ai_connection_name'] 126 | else: 127 | if 'conn_db' in node['inputs']: 128 | node['inputs']['conn_db'] = config['SQLDB_connection_name'] 129 | if 'conn' in node['inputs']: 130 | node['inputs']['conn'] = config['ACS_connection_name'] 131 | 132 | # write the yaml file back 133 | with open('./promptflow_v2/flow.dag.yaml', 'w') as f: 134 | yaml.dump(config_flow, f) 135 | 136 | # %% 137 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 6 | 7 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | 11 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 12 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 13 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /MssqlMcp/.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # Don't use tabs for indentation. 5 | [*] 6 | indent_style = space 7 | # (Please don't specify an indent_size here; that has too many unintended consequences.) 8 | 9 | # IDE0073: File header 10 | file_header_template = Copyright (c) Microsoft Corporation. All rights reserved.\nLicensed under the MIT license. 11 | 12 | # Code files 13 | [*.{cs, sql}] 14 | indent_size = 4 15 | 16 | # JSON and XML files 17 | [*.{json,xml,runsettings} ] 18 | indent_size = 2 19 | 20 | [*.cs] 21 | ### Dotnet code style settings ### 22 | 23 | # Organize usings 24 | dotnet_separate_import_directive_groups = false 25 | dotnet_sort_system_directives_first = true 26 | 27 | # Avoid "this." if not necessary 28 | dotnet_style_qualification_for_field = false:suggestion 29 | dotnet_style_qualification_for_property = false:suggestion 30 | dotnet_style_qualification_for_method = false:suggestion 31 | dotnet_style_qualification_for_event = false:suggestion 32 | 33 | # Use language keywords instead of framework type names for type references 34 | dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion 35 | dotnet_style_predefined_type_for_member_access = true:suggestion 36 | 37 | # Parentheses preferences 38 | dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:suggestion 39 | dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:suggestion 40 | dotnet_style_parentheses_in_other_operators = never_if_unnecessary:suggestion 41 | dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:suggestion 42 | 43 | # Modifier preferences 44 | dotnet_style_require_accessibility_modifiers = for_non_interface_members:suggestion 45 | 46 | # Field preferences 47 | dotnet_style_readonly_field = true:suggestion 48 | 49 | # Parameter preferences 50 | dotnet_code_quality_unused_parameters = all:suggestion 51 | 52 | # Expression-level preferences 53 | dotnet_style_coalesce_expression = true:suggestion 54 | dotnet_style_collection_initializer = true:suggestion 55 | dotnet_style_explicit_tuple_names = true:suggestion 56 | dotnet_style_null_propagation = true:suggestion 57 | dotnet_style_object_initializer = true:suggestion 58 | dotnet_style_prefer_auto_properties = true:suggestion 59 | dotnet_style_prefer_compound_assignment = true:suggestion 60 | dotnet_style_prefer_conditional_expression_over_assignment = true:suggestion 61 | dotnet_style_prefer_conditional_expression_over_return = true:suggestion 62 | dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion 63 | dotnet_style_prefer_inferred_tuple_names = true:suggestion 64 | dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion 65 | dotnet_style_prefer_simplified_interpolation = true:suggestion 66 | 67 | #### C# Coding Conventions #### 68 | 69 | # Prefer "var" everywhere 70 | csharp_style_var_for_built_in_types = true:suggestion 71 | csharp_style_var_when_type_is_apparent = true:suggestion 72 | csharp_style_var_elsewhere = true:suggestion 73 | 74 | # Prefer method-like constructs to have a block body, except for lambdas 75 | csharp_style_expression_bodied_methods = false:none 76 | csharp_style_expression_bodied_constructors = false:none 77 | csharp_style_expression_bodied_operators = false:none 78 | csharp_style_expression_bodied_local_functions = false:none 79 | csharp_style_expression_bodied_lambdas = true:none 80 | 81 | 82 | # Pattern matching preferences 83 | csharp_style_pattern_matching_over_as_with_null_check = true:suggestion 84 | csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion 85 | csharp_style_prefer_switch_expression = true:suggestion 86 | 87 | # Null-checking preferences 88 | csharp_style_conditional_delegate_call = true:suggestion 89 | 90 | # Modifier preferences 91 | csharp_prefer_static_local_function = true:suggestion 92 | csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:suggestion 93 | 94 | # Code-block preferences 95 | csharp_prefer_braces = true:error 96 | csharp_prefer_simple_using_statement = true:suggestion 97 | 98 | # Expression-level preferences 99 | csharp_style_unused_value_assignment_preference = discard_variable:suggestion 100 | csharp_prefer_simple_default_expression = true:suggestion 101 | csharp_style_deconstructed_variable_declaration = true:suggestion 102 | csharp_style_inlined_variable_declaration = true:suggestion 103 | csharp_style_pattern_local_over_anonymous_function = true:suggestion 104 | csharp_style_prefer_index_operator = true:suggestion 105 | csharp_style_prefer_range_operator = true:suggestion 106 | csharp_style_throw_expression = true:suggestion 107 | csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion 108 | 109 | # 'using' directive preferences 110 | csharp_using_directive_placement = outside_namespace:suggestion 111 | 112 | #### C# Formatting Rules #### 113 | 114 | # New line preferences 115 | csharp_new_line_before_open_brace = all 116 | csharp_new_line_before_else = true 117 | csharp_new_line_before_catch = true 118 | csharp_new_line_before_finally = true 119 | csharp_new_line_before_members_in_object_initializers = true 120 | csharp_new_line_before_members_in_anonymous_types = true 121 | csharp_new_line_between_query_expression_clauses = true 122 | 123 | # Indentation preferences 124 | csharp_indent_block_contents = true 125 | csharp_indent_braces = false 126 | csharp_indent_case_contents = true 127 | csharp_indent_case_contents_when_block = true 128 | csharp_indent_labels = no_change 129 | csharp_indent_switch_labels = true 130 | 131 | # Space preferences 132 | csharp_space_after_cast = false 133 | csharp_space_after_colon_in_inheritance_clause = true 134 | csharp_space_after_comma = true 135 | csharp_space_after_dot = false 136 | csharp_space_after_keywords_in_control_flow_statements = true 137 | csharp_space_after_semicolon_in_for_statement = true 138 | csharp_space_around_binary_operators = before_and_after 139 | csharp_space_around_declaration_statements = false 140 | csharp_space_before_colon_in_inheritance_clause = true 141 | csharp_space_before_comma = false 142 | csharp_space_before_dot = false 143 | csharp_space_before_open_square_brackets = false 144 | csharp_space_before_semicolon_in_for_statement = false 145 | csharp_space_between_empty_square_brackets = false 146 | csharp_space_between_method_call_empty_parameter_list_parentheses = false 147 | csharp_space_between_method_call_name_and_opening_parenthesis = false 148 | csharp_space_between_method_call_parameter_list_parentheses = false 149 | csharp_space_between_method_declaration_empty_parameter_list_parentheses = false 150 | csharp_space_between_method_declaration_name_and_open_parenthesis = false 151 | csharp_space_between_method_declaration_parameter_list_parentheses = false 152 | csharp_space_between_parentheses = false 153 | csharp_space_between_square_brackets = false 154 | 155 | # Non-constant fields should not be visible 156 | dotnet_diagnostic.CA2211.severity = none 157 | # Member does not access instance data and can be marked as static 158 | dotnet_diagnostic.CA1822.severity = none 159 | 160 | # Avoid dead conditional code 161 | dotnet_diagnostic.CA1508.severity = error 162 | 163 | # IDE0005: Using directive is unnecessary 164 | dotnet_diagnostic.IDE0005.severity = error 165 | # Require file header 166 | dotnet_diagnostic.IDE0073.severity = error 167 | # Fix formatting. 168 | dotnet_diagnostic.IDE0055.severity = suggestion -------------------------------------------------------------------------------- /MssqlMcp/.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | *.cmd text 3 | *.config text 4 | *.cs text 5 | *.csproj text 6 | *.editorconfig text 7 | *.gitignore text 8 | *.json text 9 | *.md text 10 | *.nuspec text 11 | *.png -text 12 | *.proj text 13 | *.prompty text 14 | *.props text 15 | *.ps1 text 16 | *.py text 17 | *.resx text 18 | *.rsp text 19 | *.runsettings text 20 | *.snk -text 21 | *.sql text 22 | *.targets text 23 | *.txt text 24 | *.xml text 25 | *.yaml text 26 | *.yml text 27 | -------------------------------------------------------------------------------- /MssqlMcp/.github/copilot-instructions.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | This project is a .NET 8 console application implementing an MCP server for Microsoft SQL Server and Azure SQL Database using the official MCP C# SDK. 4 | 5 | - Use idiomatic C# and .NET patterns. 6 | - Add unit tests for all major components. 7 | - You can find more info and examples at https://modelcontextprotocol.io/llms-full.txt -------------------------------------------------------------------------------- /MssqlMcp/.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | obj -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp.Tests/MssqlMcp.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | enable 6 | enable 7 | 8 | false 9 | true 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | runtime; build; native; contentfiles; analyzers; buildtransitive 20 | all 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp.Tests/UnitTests.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using Microsoft.Extensions.Logging; 5 | using Moq; 6 | using Mssql.McpServer; 7 | 8 | namespace MssqlMcp.Tests 9 | { 10 | public sealed class MssqlMcpTests : IDisposable 11 | { 12 | private readonly string _tableName; 13 | private readonly Tools _tools; 14 | public MssqlMcpTests() 15 | { 16 | _tableName = $"TestTable_{Guid.NewGuid():N}"; 17 | var connectionFactory = new SqlConnectionFactory(); 18 | var loggerMock = new Mock>(); 19 | _tools = new Tools(connectionFactory, loggerMock.Object); 20 | } 21 | 22 | public void Dispose() 23 | { 24 | // Cleanup: Drop the table after each test 25 | var _ = _tools.DropTable($"DROP TABLE IF EXISTS {_tableName}").GetAwaiter().GetResult(); 26 | } 27 | 28 | [Fact] 29 | public async Task CreateTable_ReturnsSuccess_WhenSqlIsValid() 30 | { 31 | var sql = $"CREATE TABLE {_tableName} (Id INT PRIMARY KEY)"; 32 | var result = await _tools.CreateTable(sql) as DbOperationResult; 33 | Assert.NotNull(result); 34 | Assert.True(result.Success); 35 | } 36 | 37 | [Fact] 38 | public async Task DescribeTable_ReturnsSchema_WhenTableExists() 39 | { 40 | // Ensure table exists 41 | var createResult = await _tools.CreateTable($"CREATE TABLE {_tableName} (Id INT PRIMARY KEY)") as DbOperationResult; 42 | Assert.NotNull(createResult); 43 | Assert.True(createResult.Success); 44 | 45 | var result = await _tools.DescribeTable(_tableName) as DbOperationResult; 46 | Assert.NotNull(result); 47 | Assert.True(result.Success); 48 | var dict = result.Data as System.Collections.IDictionary; 49 | Assert.NotNull(dict); 50 | Assert.True(dict.Contains("table")); 51 | Assert.True(dict.Contains("columns")); 52 | Assert.True(dict.Contains("indexes")); 53 | Assert.True(dict.Contains("constraints")); 54 | var table = dict["table"]; 55 | Assert.NotNull(table); 56 | var tableType = table.GetType(); 57 | Assert.NotNull(tableType.GetProperty("name")); 58 | Assert.NotNull(tableType.GetProperty("schema")); 59 | var columns = dict["columns"] as System.Collections.IEnumerable; 60 | Assert.NotNull(columns); 61 | } 62 | 63 | [Fact] 64 | public async Task DropTable_ReturnsSuccess_WhenSqlIsValid() 65 | { 66 | var sql = $"DROP TABLE IF EXISTS {_tableName}"; 67 | var result = await _tools.DropTable(sql) as DbOperationResult; 68 | Assert.NotNull(result); 69 | Assert.True(result.Success); 70 | } 71 | 72 | [Fact] 73 | public async Task InsertData_ReturnsSuccess_WhenSqlIsValid() 74 | { 75 | // Ensure table exists 76 | var createResult = await _tools.CreateTable($"CREATE TABLE {_tableName} (Id INT PRIMARY KEY)") as DbOperationResult; 77 | Assert.NotNull(createResult); 78 | Assert.True(createResult.Success); 79 | 80 | var sql = $"INSERT INTO {_tableName} (Id) VALUES (1)"; 81 | var result = await _tools.InsertData(sql) as DbOperationResult; 82 | Assert.NotNull(result); 83 | Assert.True(result.Success); 84 | Assert.True(result.RowsAffected.HasValue && result.RowsAffected.Value > 0); 85 | } 86 | 87 | [Fact] 88 | public async Task ListTables_ReturnsTables() 89 | { 90 | var result = await _tools.ListTables() as DbOperationResult; 91 | Assert.NotNull(result); 92 | Assert.True(result.Success); 93 | Assert.NotNull(result.Data); 94 | } 95 | 96 | [Fact] 97 | public async Task ReadData_ReturnsData_WhenSqlIsValid() 98 | { 99 | // Ensure table exists and has data 100 | var createResult = await _tools.CreateTable($"CREATE TABLE {_tableName} (Id INT PRIMARY KEY)") as DbOperationResult; 101 | Assert.NotNull(createResult); 102 | Assert.True(createResult.Success); 103 | var insertResult = await _tools.InsertData($"INSERT INTO {_tableName} (Id) VALUES (1)") as DbOperationResult; 104 | Assert.NotNull(insertResult); 105 | Assert.True(insertResult.Success); 106 | 107 | var sql = $"SELECT * FROM {_tableName}"; 108 | var result = await _tools.ReadData(sql) as DbOperationResult; 109 | Assert.NotNull(result); 110 | Assert.True(result.Success); 111 | Assert.NotNull(result.Data); 112 | } 113 | 114 | [Fact] 115 | public async Task UpdateData_ReturnsSuccess_WhenSqlIsValid() 116 | { 117 | // Ensure table exists and has data 118 | var createResult = await _tools.CreateTable($"CREATE TABLE {_tableName} (Id INT PRIMARY KEY)") as DbOperationResult; 119 | Assert.NotNull(createResult); 120 | Assert.True(createResult.Success); 121 | var insertResult = await _tools.InsertData($"INSERT INTO {_tableName} (Id) VALUES (1)") as DbOperationResult; 122 | Assert.NotNull(insertResult); 123 | Assert.True(insertResult.Success); 124 | 125 | var sql = $"UPDATE {_tableName} SET Id = 2 WHERE Id = 1"; 126 | var result = await _tools.UpdateData(sql) as DbOperationResult; 127 | Assert.NotNull(result); 128 | Assert.True(result.Success); 129 | Assert.True(result.RowsAffected.HasValue); 130 | } 131 | 132 | [Fact] 133 | public async Task CreateTable_ReturnsError_WhenSqlIsInvalid() 134 | { 135 | var sql = "CREATE TABLE"; 136 | var result = await _tools.CreateTable(sql) as DbOperationResult; 137 | Assert.NotNull(result); 138 | Assert.False(result.Success); 139 | Assert.Contains("syntax", result.Error ?? string.Empty, StringComparison.OrdinalIgnoreCase); 140 | } 141 | 142 | [Fact] 143 | public async Task DescribeTable_ReturnsError_WhenTableDoesNotExist() 144 | { 145 | var result = await _tools.DescribeTable("NonExistentTable") as DbOperationResult; 146 | Assert.NotNull(result); 147 | Assert.False(result.Success); 148 | Assert.Contains("Table 'NonExistentTable' not found.", result.Error ?? string.Empty, StringComparison.OrdinalIgnoreCase); 149 | } 150 | 151 | [Fact] 152 | public async Task DropTable_ReturnsError_WhenSqlIsInvalid() 153 | { 154 | var sql = "DROP"; 155 | var result = await _tools.DropTable(sql) as DbOperationResult; 156 | Assert.NotNull(result); 157 | Assert.False(result.Success); 158 | Assert.Contains("syntax", result.Error ?? string.Empty, StringComparison.OrdinalIgnoreCase); 159 | } 160 | 161 | [Fact] 162 | public async Task InsertData_ReturnsError_WhenSqlIsInvalid() 163 | { 164 | var sql = "INSERT INTO TestTable"; 165 | var result = await _tools.InsertData(sql) as DbOperationResult; 166 | Assert.NotNull(result); 167 | Assert.False(result.Success); 168 | Assert.Contains("syntax", result.Error ?? string.Empty, StringComparison.OrdinalIgnoreCase); 169 | } 170 | 171 | [Fact] 172 | public async Task ReadData_ReturnsError_WhenSqlIsInvalid() 173 | { 174 | var sql = "SELECT FROM"; 175 | var result = await _tools.ReadData(sql) as DbOperationResult; 176 | Assert.NotNull(result); 177 | Assert.False(result.Success); 178 | Assert.Contains("syntax", result.Error ?? string.Empty, StringComparison.OrdinalIgnoreCase); 179 | } 180 | 181 | [Fact] 182 | public async Task UpdateData_ReturnsError_WhenSqlIsInvalid() 183 | { 184 | var sql = "UPDATE TestTable"; 185 | var result = await _tools.UpdateData(sql) as DbOperationResult; 186 | Assert.NotNull(result); 187 | Assert.False(result.Success); 188 | Assert.Contains("syntax", result.Error ?? string.Empty, StringComparison.OrdinalIgnoreCase); 189 | } 190 | 191 | [Fact] 192 | public async Task SqlInjection_NotExecuted_When_QueryFails() 193 | { 194 | // Ensure table exists 195 | var createResult = await _tools.CreateTable($"CREATE TABLE {_tableName} (Id INT PRIMARY KEY, Name NVARCHAR(100))") as DbOperationResult; 196 | Assert.NotNull(createResult); 197 | Assert.True(createResult.Success); 198 | 199 | // Attempt SQL Injection 200 | var maliciousInput = "1; DROP TABLE " + _tableName + "; --"; 201 | var sql = $"INSERT INTO {_tableName} (Id, Name) VALUES ({maliciousInput}, 'Malicious')"; 202 | var result = await _tools.InsertData(sql) as DbOperationResult; 203 | 204 | Assert.NotNull(result); 205 | Assert.False(result.Success); 206 | Assert.Contains("syntax", result.Error ?? string.Empty, StringComparison.OrdinalIgnoreCase); 207 | 208 | // Verify table still exists 209 | var describeResult = await _tools.DescribeTable(_tableName) as DbOperationResult; 210 | Assert.NotNull(describeResult); 211 | Assert.True(describeResult.Success); 212 | } 213 | } 214 | } -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp.sln: -------------------------------------------------------------------------------- 1 | Microsoft Visual Studio Solution File, Format Version 12.00 2 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MssqlMcp.Tests", "MssqlMcp.Tests\MssqlMcp.Tests.csproj", "{06284236-507F-4730-B735-32C1A2AD87D4}" 3 | EndProject 4 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MssqlMcp", "MssqlMcp\MssqlMcp.csproj", "{6CBE2CB7-D019-4A0F-9AA5-38D18E6149DE}" 5 | EndProject 6 | Global 7 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 8 | Debug|Any CPU = Debug|Any CPU 9 | Release|Any CPU = Release|Any CPU 10 | EndGlobalSection 11 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 12 | {06284236-507F-4730-B735-32C1A2AD87D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 13 | {06284236-507F-4730-B735-32C1A2AD87D4}.Debug|Any CPU.Build.0 = Debug|Any CPU 14 | {06284236-507F-4730-B735-32C1A2AD87D4}.Release|Any CPU.ActiveCfg = Release|Any CPU 15 | {06284236-507F-4730-B735-32C1A2AD87D4}.Release|Any CPU.Build.0 = Release|Any CPU 16 | {6CBE2CB7-D019-4A0F-9AA5-38D18E6149DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 17 | {6CBE2CB7-D019-4A0F-9AA5-38D18E6149DE}.Debug|Any CPU.Build.0 = Debug|Any CPU 18 | {6CBE2CB7-D019-4A0F-9AA5-38D18E6149DE}.Release|Any CPU.ActiveCfg = Release|Any CPU 19 | {6CBE2CB7-D019-4A0F-9AA5-38D18E6149DE}.Release|Any CPU.Build.0 = Release|Any CPU 20 | EndGlobalSection 21 | GlobalSection(SolutionProperties) = preSolution 22 | HideSolutionNode = FALSE 23 | EndGlobalSection 24 | GlobalSection(ExtensibilityGlobals) = postSolution 25 | SolutionGuid = {7A848C61-1F41-4A6E-8B72-07F834E8CF1F} 26 | EndGlobalSection 27 | EndGlobal 28 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/DbOperationResult.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | namespace Mssql.McpServer; 5 | 6 | /// 7 | /// Represents the result of a database operation, including success status, error message, number of rows affected, and any returned data. 8 | /// 9 | public class DbOperationResult(bool success, string? error = null, int? rowsAffected = null, object? data = null) 10 | { 11 | /// 12 | /// Gets a value indicating whether the database operation was successful. 13 | /// 14 | public bool Success { get; } = success; 15 | 16 | /// 17 | /// Gets the error message if the operation failed; otherwise, null. 18 | /// 19 | public string? Error { get; } = error; 20 | 21 | /// 22 | /// Gets the number of rows affected by the operation, if applicable. 23 | /// 24 | public int? RowsAffected { get; } = rowsAffected; 25 | 26 | /// 27 | /// Gets any data returned by the operation, such as query results. 28 | /// 29 | public object? Data { get; } = data; 30 | } 31 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/ISqlConnectionFactory.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using Microsoft.Data.SqlClient; 5 | 6 | namespace Mssql.McpServer; 7 | 8 | /// 9 | /// Defines a factory interface for creating SQL database connections. 10 | /// 11 | public interface ISqlConnectionFactory 12 | { 13 | Task GetOpenConnectionAsync(); 14 | } -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/MssqlMcp.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | Exe 5 | net8.0 6 | Mssql.McpServer 7 | enable 8 | enable 9 | Microsoft 10 | MSSQL MCP Server 11 | 1.0.0.0 12 | 1.0.0.0 13 | 1.0.0 14 | 15 | 16 | 17 | false 18 | false 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/Program.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using Microsoft.Extensions.DependencyInjection; 5 | using Microsoft.Extensions.Hosting; 6 | using Microsoft.Extensions.Logging; 7 | 8 | namespace Mssql.McpServer; 9 | 10 | internal class Program 11 | { 12 | /// 13 | /// Entry point for the MCP server application. 14 | /// Sets up logging, configures the MCP server with standard I/O transport and tool discovery, 15 | /// builds the host, and runs the server asynchronously. 16 | /// 17 | /// Command-line arguments. 18 | private static async Task Main(string[] args) 19 | { 20 | // Create the application host builder 21 | var builder = Host.CreateApplicationBuilder(args); 22 | 23 | // Configure console logging with Trace level 24 | _ = builder.Logging.AddConsole(consoleLogOptions => 25 | { 26 | consoleLogOptions.LogToStandardErrorThreshold = LogLevel.Trace; 27 | }); 28 | 29 | // Register ISqlConnectionFactory and Tools for DI 30 | _ = builder.Services.AddSingleton(); 31 | _ = builder.Services.AddSingleton(); 32 | 33 | // Register MCP server and tools (instance-based) 34 | _ = builder.Services 35 | .AddMcpServer() 36 | .WithStdioServerTransport() 37 | .WithToolsFromAssembly(); 38 | 39 | // Build the host 40 | var host = builder.Build(); 41 | 42 | // Setup cancellation token for graceful shutdown (Ctrl+C or SIGTERM) 43 | using var cts = new CancellationTokenSource(); 44 | Console.CancelKeyPress += (sender, eventArgs) => 45 | { 46 | eventArgs.Cancel = true; // Prevent the process from terminating immediately 47 | cts.Cancel(); 48 | }; 49 | 50 | try 51 | { 52 | // Run the host with cancellation support 53 | await host.RunAsync(cts.Token); 54 | } 55 | catch (Exception ex) 56 | { 57 | // Attempt to log the exception using the host's logger 58 | if (host.Services.GetService(typeof(ILogger)) is ILogger logger) 59 | { 60 | logger.LogCritical(ex, "Unhandled exception occurred during host execution."); 61 | } 62 | else 63 | { 64 | Console.Error.WriteLine($"Unhandled exception: {ex}"); 65 | } 66 | 67 | // Set a non-zero exit code 68 | Environment.ExitCode = 1; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/SqlConnectionFactory.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using Microsoft.Data.SqlClient; 5 | 6 | namespace Mssql.McpServer; 7 | 8 | public class SqlConnectionFactory : ISqlConnectionFactory 9 | { 10 | public async Task GetOpenConnectionAsync() 11 | { 12 | var connectionString = GetConnectionString(); 13 | 14 | // Let ADO.Net handle connection pooling 15 | var conn = new SqlConnection(connectionString); 16 | await conn.OpenAsync(); 17 | return conn; 18 | } 19 | 20 | private static string GetConnectionString() 21 | { 22 | var connectionString = Environment.GetEnvironmentVariable("CONNECTION_STRING"); 23 | 24 | return string.IsNullOrEmpty(connectionString) 25 | ? throw new InvalidOperationException("Connection string is not set in the environment variable 'CONNECTION_STRING'.\n\nHINT: Have a local SQL Server, with a database called 'test', from console, run `SET CONNECTION_STRING=Server=.;Database=test;Trusted_Connection=True;TrustServerCertificate=True` and the load the .sln file") 26 | : connectionString; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/Tools/CreateTable.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using System.ComponentModel; 5 | using Microsoft.Extensions.Logging; 6 | using ModelContextProtocol.Server; 7 | 8 | namespace Mssql.McpServer; 9 | 10 | public partial class Tools 11 | { 12 | [McpServerTool( 13 | Title = "Create Table", 14 | ReadOnly = false, 15 | Destructive = false), 16 | Description("Creates a new table in the SQL Database. Expects a valid CREATE TABLE SQL statement as input.")] 17 | public async Task CreateTable( 18 | [Description("CREATE TABLE SQL statement")] string sql) 19 | { 20 | var conn = await _connectionFactory.GetOpenConnectionAsync(); 21 | try 22 | { 23 | using (conn) 24 | { 25 | using var cmd = new Microsoft.Data.SqlClient.SqlCommand(sql, conn); 26 | _ = await cmd.ExecuteNonQueryAsync(); 27 | return new DbOperationResult(success: true); 28 | } 29 | } 30 | catch (Exception ex) 31 | { 32 | _logger.LogError(ex, "CreateTable failed: {Message}", ex.Message); 33 | return new DbOperationResult(success: false, error: ex.Message); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/Tools/DescribeTable.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using System.ComponentModel; 5 | using Microsoft.Data.SqlClient; 6 | using Microsoft.Extensions.Logging; 7 | using ModelContextProtocol.Server; 8 | 9 | namespace Mssql.McpServer; 10 | 11 | public partial class Tools 12 | { 13 | [McpServerTool( 14 | Title = "Describe Table", 15 | ReadOnly = true, 16 | Idempotent = true, 17 | Destructive = false), 18 | Description("Returns table schema")] 19 | public async Task DescribeTable( 20 | [Description("Name of table")] string name) 21 | { 22 | // Query for table metadata 23 | const string TableInfoQuery = @"SELECT t.object_id AS id, t.name, s.name AS [schema], p.value AS description, t.type, u.name AS owner 24 | FROM sys.tables t 25 | INNER JOIN sys.schemas s ON t.schema_id = s.schema_id 26 | LEFT JOIN sys.extended_properties p ON p.major_id = t.object_id AND p.minor_id = 0 AND p.name = 'MS_Description' 27 | LEFT JOIN sys.sysusers u ON t.principal_id = u.uid 28 | WHERE t.name = @TableName"; 29 | 30 | // Query for columns 31 | const string ColumnsQuery = @"SELECT c.name, ty.name AS type, c.max_length AS length, c.precision, c.is_nullable AS nullable, p.value AS description 32 | FROM sys.columns c 33 | INNER JOIN sys.types ty ON c.user_type_id = ty.user_type_id 34 | LEFT JOIN sys.extended_properties p ON p.major_id = c.object_id AND p.minor_id = c.column_id AND p.name = 'MS_Description' 35 | WHERE c.object_id = (SELECT object_id FROM sys.tables WHERE name = @TableName)"; 36 | 37 | // Query for indexes 38 | const string IndexesQuery = @"SELECT i.name, i.type_desc AS type, p.value AS description, 39 | STUFF((SELECT ',' + c.name FROM sys.index_columns ic 40 | INNER JOIN sys.columns c ON ic.object_id = c.object_id AND ic.column_id = c.column_id 41 | WHERE ic.object_id = i.object_id AND ic.index_id = i.index_id ORDER BY ic.key_ordinal FOR XML PATH('')), 1, 1, '') AS keys 42 | FROM sys.indexes i 43 | LEFT JOIN sys.extended_properties p ON p.major_id = i.object_id AND p.minor_id = i.index_id AND p.name = 'MS_Description' 44 | WHERE i.object_id = (SELECT object_id FROM sys.tables WHERE name = @TableName) AND i.is_primary_key = 0 AND i.is_unique_constraint = 0"; 45 | 46 | // Query for constraints 47 | const string ConstraintsQuery = @"SELECT kc.name, kc.type_desc AS type, 48 | STUFF((SELECT ',' + c.name FROM sys.index_columns ic 49 | INNER JOIN sys.columns c ON ic.object_id = c.object_id AND ic.column_id = c.column_id 50 | WHERE ic.object_id = kc.parent_object_id AND ic.index_id = kc.unique_index_id ORDER BY ic.key_ordinal FOR XML PATH('')), 1, 1, '') AS keys 51 | FROM sys.key_constraints kc 52 | WHERE kc.parent_object_id = (SELECT object_id FROM sys.tables WHERE name = @TableName)"; 53 | 54 | var conn = await _connectionFactory.GetOpenConnectionAsync(); 55 | try 56 | { 57 | using (conn) 58 | { 59 | var result = new Dictionary(); 60 | // Table info 61 | using (var cmd = new SqlCommand(TableInfoQuery, conn)) 62 | { 63 | var _ = cmd.Parameters.AddWithValue("@TableName", name); 64 | using var reader = await cmd.ExecuteReaderAsync(); 65 | if (await reader.ReadAsync()) 66 | { 67 | result["table"] = new 68 | { 69 | id = reader["id"], 70 | name = reader["name"], 71 | schema = reader["schema"], 72 | owner = reader["owner"], 73 | type = reader["type"], 74 | description = reader["description"] is DBNull ? null : reader["description"] 75 | }; 76 | } 77 | else 78 | { 79 | return new DbOperationResult(success: false, error: $"Table '{name}' not found."); 80 | } 81 | } 82 | // Columns 83 | using (var cmd = new SqlCommand(ColumnsQuery, conn)) 84 | { 85 | var _ = cmd.Parameters.AddWithValue("@TableName", name); 86 | using var reader = await cmd.ExecuteReaderAsync(); 87 | var columns = new List(); 88 | while (await reader.ReadAsync()) 89 | { 90 | columns.Add(new 91 | { 92 | name = reader["name"], 93 | type = reader["type"], 94 | length = reader["length"], 95 | precision = reader["precision"], 96 | nullable = (bool)reader["nullable"], 97 | description = reader["description"] is DBNull ? null : reader["description"] 98 | }); 99 | } 100 | result["columns"] = columns; 101 | } 102 | // Indexes 103 | using (var cmd = new SqlCommand(IndexesQuery, conn)) 104 | { 105 | var _ = cmd.Parameters.AddWithValue("@TableName", name); 106 | using var reader = await cmd.ExecuteReaderAsync(); 107 | var indexes = new List(); 108 | while (await reader.ReadAsync()) 109 | { 110 | indexes.Add(new 111 | { 112 | name = reader["name"], 113 | type = reader["type"], 114 | description = reader["description"] is DBNull ? null : reader["description"], 115 | keys = reader["keys"] 116 | }); 117 | } 118 | result["indexes"] = indexes; 119 | } 120 | // Constraints 121 | using (var cmd = new SqlCommand(ConstraintsQuery, conn)) 122 | { 123 | var _ = cmd.Parameters.AddWithValue("@TableName", name); 124 | using var reader = await cmd.ExecuteReaderAsync(); 125 | var constraints = new List(); 126 | while (await reader.ReadAsync()) 127 | { 128 | constraints.Add(new 129 | { 130 | name = reader["name"], 131 | type = reader["type"], 132 | keys = reader["keys"] 133 | }); 134 | } 135 | result["constraints"] = constraints; 136 | } 137 | return new DbOperationResult(success: true, data: result); 138 | } 139 | } 140 | catch (Exception ex) 141 | { 142 | _logger.LogError(ex, "DescribeTable failed: {Message}", ex.Message); 143 | return new DbOperationResult(success: false, error: ex.Message); 144 | } 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/Tools/DropTable.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using System.ComponentModel; 5 | using Microsoft.Extensions.Logging; 6 | using ModelContextProtocol.Server; 7 | 8 | namespace Mssql.McpServer; 9 | 10 | public partial class Tools 11 | { 12 | [McpServerTool( 13 | Title = "Drop Table", 14 | ReadOnly = false, 15 | Destructive = true), 16 | Description("Drops a table in the SQL Database. Expects a valid DROP TABLE SQL statement as input.")] 17 | public async Task DropTable( 18 | [Description("DROP TABLE SQL statement")] string sql) 19 | { 20 | var conn = await _connectionFactory.GetOpenConnectionAsync(); 21 | try 22 | { 23 | using (conn) 24 | { 25 | using var cmd = new Microsoft.Data.SqlClient.SqlCommand(sql, conn); 26 | _ = await cmd.ExecuteNonQueryAsync(); 27 | return new DbOperationResult(success: true); 28 | } 29 | } 30 | catch (Exception ex) 31 | { 32 | _logger.LogError(ex, "DropTable failed: {Message}", ex.Message); 33 | return new DbOperationResult(success: false, error: ex.Message); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/Tools/InsertData.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using System.ComponentModel; 5 | using Microsoft.Extensions.Logging; 6 | using ModelContextProtocol.Server; 7 | 8 | namespace Mssql.McpServer; 9 | 10 | public partial class Tools 11 | { 12 | [McpServerTool( 13 | Title = "Insert Data", 14 | ReadOnly = false, 15 | Destructive = false), 16 | Description("Updates data in a table in the SQL Database. Expects a valid INSERT SQL statement as input. ")] 17 | public async Task InsertData( 18 | [Description("INSERT SQL statement")] string sql) 19 | { 20 | var conn = await _connectionFactory.GetOpenConnectionAsync(); 21 | try 22 | { 23 | using (conn) 24 | { 25 | using var cmd = new Microsoft.Data.SqlClient.SqlCommand(sql, conn); 26 | var rows = await cmd.ExecuteNonQueryAsync(); 27 | return new DbOperationResult(success: true, rowsAffected: rows); 28 | } 29 | } 30 | catch (Exception ex) 31 | { 32 | _logger.LogError(ex, "InsertData failed: {Message}", ex.Message); 33 | return new DbOperationResult(success: false, error: ex.Message); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/Tools/ListTables.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using System.ComponentModel; 5 | using Microsoft.Data.SqlClient; 6 | using Microsoft.Extensions.Logging; 7 | using ModelContextProtocol.Server; 8 | 9 | namespace Mssql.McpServer; 10 | 11 | public partial class Tools 12 | { 13 | private const string ListTablesQuery = @"SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_SCHEMA, TABLE_NAME"; 14 | 15 | [McpServerTool( 16 | Title = "List Tables", 17 | ReadOnly = true, 18 | Idempotent = true, 19 | Destructive = false), 20 | Description("Lists all tables in the SQL Database.")] 21 | public async Task ListTables() 22 | { 23 | var conn = await _connectionFactory.GetOpenConnectionAsync(); 24 | try 25 | { 26 | using (conn) 27 | { 28 | using var cmd = new SqlCommand(ListTablesQuery, conn); 29 | var tables = new List(); 30 | using var reader = await cmd.ExecuteReaderAsync(); 31 | while (await reader.ReadAsync()) 32 | { 33 | tables.Add($"{reader.GetString(0)}.{reader.GetString(1)}"); 34 | } 35 | return new DbOperationResult(success: true, data: tables); 36 | } 37 | } 38 | catch (Exception ex) 39 | { 40 | _logger.LogError(ex, "ListTables failed: {Message}", ex.Message); 41 | return new DbOperationResult(success: false, error: ex.Message); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/Tools/ReadData.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using System.ComponentModel; 5 | using Microsoft.Data.SqlClient; 6 | using Microsoft.Extensions.Logging; 7 | using ModelContextProtocol.Server; 8 | 9 | namespace Mssql.McpServer; 10 | public partial class Tools 11 | { 12 | [McpServerTool( 13 | Title = "Read Data", 14 | ReadOnly = true, 15 | Idempotent = true, 16 | Destructive = false), 17 | Description("Executes SQL queries against SQL Database to read data")] 18 | public async Task ReadData( 19 | [Description("SQL query to execute")] string sql) 20 | { 21 | var conn = await _connectionFactory.GetOpenConnectionAsync(); 22 | try 23 | { 24 | using (conn) 25 | { 26 | using var cmd = new SqlCommand(sql, conn); 27 | using var reader = await cmd.ExecuteReaderAsync(); 28 | var results = new List>(); 29 | while (await reader.ReadAsync()) 30 | { 31 | var row = new Dictionary(); 32 | for (var i = 0; i < reader.FieldCount; i++) 33 | { 34 | row[reader.GetName(i)] = reader.IsDBNull(i) ? null : reader.GetValue(i); 35 | } 36 | results.Add(row); 37 | } 38 | return new DbOperationResult(success: true, data: results); 39 | } 40 | } 41 | catch (Exception ex) 42 | { 43 | _logger.LogError(ex, "ReadData failed: {Message}", ex.Message); 44 | return new DbOperationResult(success: false, error: ex.Message); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/Tools/Tools.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using System.Data; 5 | using Microsoft.Extensions.Logging; 6 | using ModelContextProtocol.Server; 7 | 8 | namespace Mssql.McpServer; 9 | 10 | // Register this class as a tool container 11 | [McpServerToolType] 12 | public partial class Tools(ISqlConnectionFactory connectionFactory, ILogger logger) 13 | { 14 | private readonly ISqlConnectionFactory _connectionFactory = connectionFactory; 15 | private readonly ILogger _logger = logger; 16 | 17 | // Helper to convert DataTable to a serializable list 18 | private static List> DataTableToList(DataTable table) 19 | { 20 | var result = new List>(); 21 | foreach (DataRow row in table.Rows) 22 | { 23 | var dict = new Dictionary(); 24 | foreach (DataColumn col in table.Columns) 25 | { 26 | dict[col.ColumnName] = row[col]; 27 | } 28 | result.Add(dict); 29 | } 30 | return result; 31 | } 32 | } -------------------------------------------------------------------------------- /MssqlMcp/MssqlMcp/Tools/UpdateData.cs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. All rights reserved. 2 | // Licensed under the MIT license. 3 | 4 | using System.ComponentModel; 5 | using Microsoft.Extensions.Logging; 6 | using ModelContextProtocol.Server; 7 | 8 | namespace Mssql.McpServer; 9 | 10 | public partial class Tools 11 | { 12 | [McpServerTool( 13 | Title = "Update Data", 14 | ReadOnly = false, 15 | Destructive = true), 16 | Description("Updates data in a table in the SQL Database. Expects a valid UPDATE SQL statement as input.")] 17 | public async Task UpdateData( 18 | [Description("UPDATE SQL statement")] string sql) 19 | { 20 | var conn = await _connectionFactory.GetOpenConnectionAsync(); 21 | try 22 | { 23 | using (conn) 24 | { 25 | using var cmd = new Microsoft.Data.SqlClient.SqlCommand(sql, conn); 26 | var rows = await cmd.ExecuteNonQueryAsync(); 27 | return new DbOperationResult(true, null, rows); 28 | } 29 | } 30 | catch (Exception ex) 31 | { 32 | _logger.LogError(ex, "UpdateData failed: {Message}", ex.Message); 33 | return new DbOperationResult(false, ex.Message); 34 | } 35 | } 36 | } 37 | 38 | -------------------------------------------------------------------------------- /MssqlMcp/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Mssql SQL MCP Server (.NET 8) 3 | 4 | This project is a .NET 8 console application implementing a Model Context Protocol (MCP) server for MSSQL Databases using the official [MCP C# SDK](https://github.com/modelcontextprotocol/csharp-sdk). 5 | 6 | ## Features 7 | 8 | - Provide connection string via environment variable `CONNECTION_STRING`. 9 | - **MCP Tools Implemented**: 10 | - ListTables: List all tables in the database. 11 | - DescribeTable: Get schema/details for a table. 12 | - CreateTable: Create new tables. 13 | - DropTable: Drop existing tables. 14 | - InsertData: Insert data into tables. 15 | - ReadData: Read/query data from tables. 16 | - UpdateData: Update values in tables. 17 | - **Logging**: Console logging using Microsoft.Extensions.Logging. 18 | - **Unit Tests**: xUnit-based unit tests for all major components. 19 | 20 | ## Getting Started 21 | 22 | ### Prerequisites 23 | 24 | - Access to a SQL Server or Azure SQL Database 25 | 26 | ### Setup 27 | 28 | 1. **Build *** 29 | 30 | --- 31 | ```sh 32 | cd MssqlMcp 33 | dotnet build 34 | ``` 35 | --- 36 | 37 | 38 | 2. VSCode: **Start VSCode, and add MCP Server config to VSCode Settings** 39 | 40 | Load the settings file in VSCode (Ctrl+Shift+P > Preferences: Open Settings (JSON)). 41 | 42 | Add a new MCP Server with the following settings: 43 | 44 | --- 45 | ```json 46 | "MSSQL MCP": { 47 | "type": "stdio", 48 | "command": "C:\\src\\MssqlMcp\\MssqlMcp\\bin\\Debug\\net8.0\\MssqlMcp.exe", 49 | "env": { 50 | "CONNECTION_STRING": "Server=.;Database=test;Trusted_Connection=True;TrustServerCertificate=True" 51 | } 52 | } 53 | ``` 54 | --- 55 | 56 | NOTE: Replace the path "C:\\src\\SQL-AI-samples" with the location of your SQL-AI-samples repo on your machine. 57 | 58 | e.g. your MCP settings should look like this if "MSSQL MCP" is your own MCP Server in VSCode settings: 59 | 60 | --- 61 | ```json 62 | "mcp": { 63 | "servers": { 64 | "MSSQL MCP": { 65 | "type": "stdio", 66 | "command": "C:\\src\\SQL-AI-samples\\MssqlMcp\\MssqlMcp\\bin\\Debug\\net8.0\\MssqlMcp.exe", 67 | "env": { 68 | "CONNECTION_STRING": "Server=.;Database=test;Trusted_Connection=True;TrustServerCertificate=True" 69 | } 70 | } 71 | } 72 | ``` 73 | --- 74 | 75 | An example of using a connection string for Azure SQL Database: 76 | --- 77 | ```json 78 | "mcp": { 79 | "servers": { 80 | "MSSQL MCP": { 81 | "type": "stdio", 82 | "command": "C:\\src\\SQL-AI-samples\\MssqlMcp\\MssqlMcp\\bin\\Debug\\net8.0\\MssqlMcp.exe", 83 | "env": { 84 | "CONNECTION_STRING": "Server=tcp:.database.windows.net,1433;Initial Catalog=;Encrypt=Mandatory;TrustServerCertificate=False;Connection Timeout=30;Authentication=Active Directory Interactive" 85 | } 86 | } 87 | } 88 | ``` 89 | --- 90 | 91 | **Run the MCP Server** 92 | 93 | Save the Settings file, and then you should see the "Start" button appear in the settings.json. Click "Start" to start the MCP Server. (You can then click on "Running" to view the Output window). 94 | 95 | Start Chat (Ctrl+Shift+I), make sure Agent Mode is selected. 96 | 97 | Click the tools icon, and ensure the "MSSQL MCP" tools are selected. 98 | 99 | Then type in the chat window "List tables in the database" and hit enter. (If you have other tools loaded, you may need to specify "MSSQL MCP" in the initial prompt, e.g. "Using MSSQL MCP, list tables"). 100 | 101 | 3. Claude Desktop: **Add MCP Server config to Claude Desktop** 102 | 103 | Press File > Settings > Developer. 104 | Press the "Edit Config" button (which will load the claude_desktop_config.json file in your editor). 105 | 106 | Add a new MCP Server with the following settings: 107 | 108 | --- 109 | ```json 110 | { 111 | "mcpServers": { 112 | "MSSQL MCP": { 113 | "command": "C:\\src\\SQL-AI-samples\\MssqlMcp\\MssqlMcp\\bin\\Debug\\net8.0\\MssqlMcp.exe", 114 | "env": { 115 | "CONNECTION_STRING": "Server=.;Database=test;Trusted_Connection=True;TrustServerCertificate=True" 116 | } 117 | } 118 | } 119 | } 120 | ``` 121 | --- 122 | 123 | Save the file, start a new Chat, you'll see the "Tools" icon, it should list 7 MSSQL MCP tools. 124 | 125 | # Troubleshooting 126 | 127 | 1. If you get a "Task canceled" error using "Active Directory Default", try "Active Directory Interactive". 128 | 129 | 130 | 131 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- 1 | # Support 2 | 3 | ## How to file issues and get help 4 | 5 | This project uses GitHub Issues to track bugs and feature requests. Please search the existing 6 | issues before filing new issues to avoid duplicates. For new issues, file your bug or 7 | feature request as a new Issue. 8 | 9 | ## Microsoft Support Policy 10 | 11 | Support for this **PROJECT or PRODUCT** is limited to the resources listed above. 12 | --------------------------------------------------------------------------------