├── .devcontainer ├── devcontainer.json └── script.sh ├── .github ├── CODE_OF_CONDUCT.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── images ├── Who_is_Sofia.png ├── application.png ├── codespace-create.png ├── codespace_secrets.png ├── diagram.png └── grafana_tempo.png ├── infra ├── acr.tf ├── aks.tf ├── identity.tf ├── installation_script.tftpl ├── microservices-grafana-values.yaml ├── microservices-tempo-values.yaml ├── network.tf ├── openai.tf ├── output.tf ├── providers.tf ├── resource_group.tf └── variables.tf └── sample-application ├── .gitignore ├── Dockerfile ├── chatbot.py └── requirements.txt /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Terraform AzureCLI and Kubectl", 3 | "postAttachCommand": "bash .devcontainer/script.sh", 4 | "customizations": { 5 | "vscode": { 6 | "extensions": [ 7 | "hashicorp.terraform", 8 | "ms-python.python", 9 | "ms-azuretools.vscode-docker" 10 | ] 11 | } 12 | }, 13 | "features": { 14 | "azure-cli": "latest", 15 | "terraform": { 16 | "version": "latest", 17 | "tflint": "latest", 18 | "terragrunt": "latest" 19 | }, 20 | "kubectl-helm-minikube": { 21 | "version": "latest", 22 | "helm": "latest", 23 | "minikube": "latest" 24 | }, 25 | "ghcr.io/dhoeric/features/stern:1": {}, 26 | "ghcr.io/devcontainers-contrib/features/kubectx-kubens:1": {} 27 | } 28 | } -------------------------------------------------------------------------------- /.devcontainer/script.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$ARM_CLIENT_ID" ]; then 4 | echo "ARM_CLIENT_ID is not set" 5 | exit 0 6 | fi 7 | 8 | # Check if $ARM_CLIENT_SECRET exists 9 | if [ -z "$ARM_CLIENT_SECRET" ]; then 10 | echo "ARM_CLIENT_SECRET is not set" 11 | exit 0 12 | fi 13 | 14 | # Check if $ARM_TENANT_ID exists 15 | if [ -z "$ARM_TENANT_ID" ]; then 16 | echo "ARM_TENANT_ID is not set" 17 | exit 0 18 | fi 19 | 20 | # Check if $ARM_SUBSCRIPTION_ID exists 21 | if [ -z "$ARM_SUBSCRIPTION_ID" ]; then 22 | echo "ARM_SUBSCRIPTION_ID is not set" 23 | exit 0 24 | fi 25 | 26 | az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | > Please provide us with the following information: 5 | > --------------------------------------------------------------- 6 | 7 | ### This issue is for a: (mark with an `x`) 8 | ``` 9 | - [ ] bug report -> please search issues before submitting 10 | - [ ] feature request 11 | - [ ] documentation issue or request 12 | - [ ] regression (a behavior that used to work and stopped in a new release) 13 | ``` 14 | 15 | ### Minimal steps to reproduce 16 | > 17 | 18 | ### Any log messages given by the failure 19 | > 20 | 21 | ### Expected/desired behavior 22 | > 23 | 24 | ### OS and Version? 25 | > Windows 7, 8 or 10. Linux (which distribution). macOS (Yosemite? El Capitan? Sierra?) 26 | 27 | ### Versions 28 | > 29 | 30 | ### Mention any other details that might be useful 31 | 32 | > --------------------------------------------------------------- 33 | > Thanks! We'll be in touch soon. 34 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | 3 | * ... 4 | 5 | ## Does this introduce a breaking change? 6 | 7 | ``` 8 | [ ] Yes 9 | [ ] No 10 | ``` 11 | 12 | ## Pull Request Type 13 | What kind of change does this Pull Request introduce? 14 | 15 | 16 | ``` 17 | [ ] Bugfix 18 | [ ] Feature 19 | [ ] Code style update (formatting, local variables) 20 | [ ] Refactoring (no functional changes, no api changes) 21 | [ ] Documentation content changes 22 | [ ] Other... Please describe: 23 | ``` 24 | 25 | ## How to Test 26 | * Get the code 27 | 28 | ``` 29 | git clone [repo-address] 30 | cd [repo-name] 31 | git checkout [branch-name] 32 | npm install 33 | ``` 34 | 35 | * Test the code 36 | 37 | ``` 38 | ``` 39 | 40 | ## What to Check 41 | Verify that the following are valid 42 | * ... 43 | 44 | ## Other Information 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | terraform.tfstate 2 | terraform.tfstate.backup 3 | .terraform.lock.hcl 4 | .terraform -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [project-title] Changelog 2 | 3 | 4 | # x.y.z (yyyy-mm-dd) 5 | 6 | *Features* 7 | * ... 8 | 9 | *Bug Fixes* 10 | * ... 11 | 12 | *Breaking Changes* 13 | * ... 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to [project-title] 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 6 | 7 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | 11 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 12 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 13 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 14 | 15 | - [Code of Conduct](#coc) 16 | - [Issues and Bugs](#issue) 17 | - [Feature Requests](#feature) 18 | - [Submission Guidelines](#submit) 19 | 20 | ## Code of Conduct 21 | Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 22 | 23 | ## Found an Issue? 24 | If you find a bug in the source code or a mistake in the documentation, you can help us by 25 | [submitting an issue](#submit-issue) to the GitHub Repository. Even better, you can 26 | [submit a Pull Request](#submit-pr) with a fix. 27 | 28 | ## Want a Feature? 29 | You can *request* a new feature by [submitting an issue](#submit-issue) to the GitHub 30 | Repository. If you would like to *implement* a new feature, please submit an issue with 31 | a proposal for your work first, to be sure that we can use it. 32 | 33 | * **Small Features** can be crafted and directly [submitted as a Pull Request](#submit-pr). 34 | 35 | ## Submission Guidelines 36 | 37 | ### Submitting an Issue 38 | Before you submit an issue, search the archive, maybe your question was already answered. 39 | 40 | If your issue appears to be a bug, and hasn't been reported, open a new issue. 41 | Help us to maximize the effort we can spend fixing issues and adding new 42 | features, by not reporting duplicate issues. Providing the following information will increase the 43 | chances of your issue being dealt with quickly: 44 | 45 | * **Overview of the Issue** - if an error is being thrown a non-minified stack trace helps 46 | * **Version** - what version is affected (e.g. 0.1.2) 47 | * **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you 48 | * **Browsers and Operating System** - is this a problem with all browsers? 49 | * **Reproduce the Error** - provide a live example or a unambiguous set of steps 50 | * **Related Issues** - has a similar issue been reported before? 51 | * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be 52 | causing the problem (line of code or commit) 53 | 54 | You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/[organization-name]/[repository-name]/issues/new]. 55 | 56 | ### Submitting a Pull Request (PR) 57 | Before you submit your Pull Request (PR) consider the following guidelines: 58 | 59 | * Search the repository (https://github.com/[organization-name]/[repository-name]/pulls) for an open or closed PR 60 | that relates to your submission. You don't want to duplicate effort. 61 | 62 | * Make your changes in a new git fork: 63 | 64 | * Commit your changes using a descriptive commit message 65 | * Push your fork to GitHub: 66 | * In GitHub, create a pull request 67 | * If we suggest changes then: 68 | * Make the required updates. 69 | * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): 70 | 71 | ```shell 72 | git rebase master -i 73 | git push -f 74 | ``` 75 | 76 | That's it! Thank you for your contribution! 77 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Azure Samples 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure OpenAI Terraform deployment for sample chatbot 2 | 3 | This sample application deploys an AI-powered document search using Azure OpenAI Service, Azure Kubernetes Service (AKS), and a Python application leveraging the [Llama index](https://gpt-index.readthedocs.io/en/latest/) ans [Streamlit](https://docs.streamlit.io/library/get-started). The application will be deployed within a virtual network to ensure security and isolation. Users will be able to upload documents and ask questions based on the content of the uploaded documents. 4 | 5 | ![diagram](./images/diagram.png) 6 | 7 | ## Prerequisites 8 | 9 | - Azure subscription. If you don't have an Azure subscription, [create a free account](https://azure.microsoft.com/free/?ref=microsoft.com&utm_source=microsoft.com&utm_medium=docs&utm_campaign=visualstudio) before you begin. 10 | - Subscription access to Azure OpenAI service. Request Access to Azure OpenAI Service [here](https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu). 11 | - [Terraform](https://learn.microsoft.com/azure/developer/terraform/quickstart-configure). 12 | 13 | The easiest way to run this sample is to run it creating a new [GitHub Codespace](https://codespaces.new/Azure-Samples/azure-openai-terraform-deployment-sample) 14 | 15 | [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/Azure-Samples/azure-openai-terraform-deployment-sample) 16 | 17 | ## Quickstart 18 | 19 | ### (Optional) configure GitHub Codespaces secrets to access your Azure subscription 20 | 21 | - Run the following command to create a service principal with the "Owner" role for a specific subscription, and outputs its information in JSON format. 22 | 23 | ```bash 24 | az ad sp create-for-rbac --role="Owner" --scopes="/subscriptions/" -o json 25 | ``` 26 | 27 | - In your github account go to Codespaces and Create a new Codespace with "Azure-Sample/azure-openai-terraform-deployment-sample" repository and select the main branch. 28 | 29 | ![codespace_create](./images/codespace-create.png) 30 | 31 | - In your github account, go to Settings. On the left pane, select Codespaces tab and create a secret for `ARM_CLIENT_ID`, `ARM_CLIENT_SECRET`, `ARM_SUBSCRIPTION_ID` and `ARM_TENANT_ID` values, as shown in the image below. For each secret, on the Repository access section, click on the "Select repositories" dropdown menu and select "Azure-Samples/azure-openai-terraform-deployment-sample". 32 | 33 | ![codespace_secrets](./images/codespace_secrets.png) 34 | 35 | - Follow this link to create a new [GitHub Codespace](https://codespaces.new/Azure-Samples/azure-openai-terraform-deployment-sample). 36 | 37 | [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/Azure-Samples/azure-openai-terraform-deployment-sample) 38 | 39 | ### Run the Terraform 40 | 41 | - Clone or fork this repository. (Skip if using GitHub codespaces) 42 | ``` 43 | git clone https://github.com/Azure-Samples/azure-openai-terraform-deployment-sample/ 44 | cd azure-openai-terraform-deployment-sample 45 | ``` 46 | 47 | - Go to the `infra` folder and run the following command to initialize your working directory. 48 | 49 | ```bash 50 | cd infra 51 | terraform init 52 | ``` 53 | 54 | - Run terraform apply to deploy all the necessary resources on Azure. 55 | 56 | ```bash 57 | terraform apply 58 | ``` 59 | 60 | - Run the following command. This script retrieves the AKS cluster credentials, logs in to the ACR, builds and pushes a Docker image, creates a federated identity, and deploys resources to the Kubernetes cluster using a YAML manifest. 61 | 62 | ```bash 63 | terraform output -raw installation-script | bash 64 | ``` 65 | 66 | - Get the external ip address of the service by running the command bellow. 67 | 68 | ```bash 69 | kubectl get services -n chatbot 70 | ``` 71 | 72 | - Copy the external ip address and paste it in your browser. The application should load in a few seconds. 73 | 74 | ![app](/images/application.png) 75 | 76 | - You can test the application by uploading a sample `file.txt` with the following content: 77 | 78 | ``` 79 | Saverio and Sofia both work at Microsoft. 80 | They live in Switzerland. 81 | Saverio is Italian. 82 | Sofia is from Portugal. 83 | ``` 84 | 85 | And then ask a question like the following screenshot: 86 | 87 | ![app](/images/Who_is_Sofia.png) 88 | 89 | - Access the Grafana dashboard by running the following command. 90 | 91 | ```bash 92 | kubectl port-forward svc/grafana 8080:80 93 | ``` 94 | And point your browser to `http://localhost:8080`. 95 | 96 | You can browse to the metrics explorer and see the traces collected by the application. In the traces you are able to inspect the actual prompt that the llama_index library is using to query the OpenAI service. 97 | 98 | ![app](/images/grafana_tempo.png) 99 | 100 | ## Resources 101 | 102 | - [Azure OpenAI](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/overview) 103 | - [Azure OpenAI Terraform verified module](https://registry.terraform.io/modules/Azure/openai/azurerm/latest). 104 | -------------------------------------------------------------------------------- /images/Who_is_Sofia.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-openai-terraform-deployment-sample/b5a113691e19f23667f2caf268c5d4916d370de6/images/Who_is_Sofia.png -------------------------------------------------------------------------------- /images/application.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-openai-terraform-deployment-sample/b5a113691e19f23667f2caf268c5d4916d370de6/images/application.png -------------------------------------------------------------------------------- /images/codespace-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-openai-terraform-deployment-sample/b5a113691e19f23667f2caf268c5d4916d370de6/images/codespace-create.png -------------------------------------------------------------------------------- /images/codespace_secrets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-openai-terraform-deployment-sample/b5a113691e19f23667f2caf268c5d4916d370de6/images/codespace_secrets.png -------------------------------------------------------------------------------- /images/diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-openai-terraform-deployment-sample/b5a113691e19f23667f2caf268c5d4916d370de6/images/diagram.png -------------------------------------------------------------------------------- /images/grafana_tempo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure-Samples/azure-openai-terraform-deployment-sample/b5a113691e19f23667f2caf268c5d4916d370de6/images/grafana_tempo.png -------------------------------------------------------------------------------- /infra/acr.tf: -------------------------------------------------------------------------------- 1 | resource "random_string" "acr_suffix" { 2 | length = 8 3 | upper = false 4 | numeric = true 5 | special = false 6 | } 7 | 8 | resource "azurerm_container_registry" "acr" { 9 | location = azurerm_resource_group.this.location 10 | name = "openai${random_string.acr_suffix.result}" 11 | resource_group_name = azurerm_resource_group.this.name 12 | sku = "Premium" 13 | 14 | retention_policy { 15 | days = 1 16 | enabled = true 17 | } 18 | } -------------------------------------------------------------------------------- /infra/aks.tf: -------------------------------------------------------------------------------- 1 | module "aks" { 2 | source = "Azure/aks/azurerm" 3 | version = "9.1.0" 4 | resource_group_name = azurerm_resource_group.this.name 5 | kubernetes_version = null # will install the latest version 6 | orchestrator_version = null # will install the latest version 7 | prefix = "openai" 8 | network_plugin = "azure" 9 | vnet_subnet_id = lookup(module.vnet.vnet_subnets_name_id, "subnet0") 10 | os_disk_size_gb = 50 11 | sku_tier = "Standard" 12 | role_based_access_control_enabled = true 13 | rbac_aad = false 14 | private_cluster_enabled = false 15 | enable_auto_scaling = true 16 | enable_host_encryption = false 17 | log_analytics_workspace_enabled = false 18 | agents_min_count = 1 19 | agents_max_count = 5 20 | agents_count = null # Please set `agents_count` `null` while `enable_auto_scaling` is `true` to avoid possible `agents_count` changes. 21 | agents_max_pods = 100 22 | agents_pool_name = "system" 23 | agents_availability_zones = ["1", "2"] 24 | agents_type = "VirtualMachineScaleSets" 25 | agents_size = "Standard_D3_v2" 26 | workload_identity_enabled = true 27 | oidc_issuer_enabled = true 28 | 29 | agents_labels = { 30 | "nodepool" : "defaultnodepool" 31 | } 32 | 33 | agents_tags = { 34 | "Agent" : "defaultnodepoolagent" 35 | } 36 | 37 | attached_acr_id_map = { 38 | acr = azurerm_container_registry.acr.id 39 | } 40 | 41 | network_policy = "azure" 42 | net_profile_dns_service_ip = "10.0.0.10" 43 | net_profile_service_cidr = "10.0.0.0/16" 44 | 45 | key_vault_secrets_provider_enabled = true 46 | secret_rotation_enabled = true 47 | secret_rotation_interval = "3m" 48 | 49 | depends_on = [module.vnet] 50 | } 51 | -------------------------------------------------------------------------------- /infra/identity.tf: -------------------------------------------------------------------------------- 1 | resource azurerm_user_assigned_identity chatbot { 2 | name = "chatbot" 3 | resource_group_name = azurerm_resource_group.this.name 4 | location = azurerm_resource_group.this.location 5 | } 6 | 7 | # Assign the "Cognitive Services User" to the chatbot User Assigned identity 8 | resource "azurerm_role_assignment" "chatbot_role_assignment" { 9 | scope = azurerm_resource_group.this.id 10 | role_definition_name = "Cognitive Services User" 11 | principal_id = azurerm_user_assigned_identity.chatbot.principal_id 12 | } -------------------------------------------------------------------------------- /infra/installation_script.tftpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | az aks get-credentials --resource-group ${resourceGroup} --name ${aksName} --overwrite-existing 3 | az acr build --registry ${registry} --image chatbot:latest ./../sample-application 4 | 5 | az identity federated-credential create --name chatbotfederatedidentity --identity-name chatbot --resource-group ${resourceGroup} --issuer ${oidc_url} --subject system:serviceaccount:chatbot:chatbot 6 | 7 | # Install Grafana and Grafana Tempo in the default namespace 8 | helm repo add grafana https://grafana.github.io/helm-charts 9 | helm repo update 10 | helm upgrade -f microservices-tempo-values.yaml --install tempo grafana/tempo-distributed 11 | helm upgrade -f microservices-grafana-values.yaml --install grafana grafana/grafana 12 | 13 | kubectl create namespace chatbot --dry-run=client -o yaml | kubectl apply -f - 14 | kubectl apply -f - <