├── CHANGELOG.md ├── Images ├── DrillDown1.PNG ├── drilldown2.png ├── architecture.png ├── mytimetrigger.png ├── visualization.png ├── az-cli-version.png ├── dataingestion_1.png ├── dataingestion_2.png ├── dataingestion_3.png ├── dataingestion_4.png ├── dataingestion_5.png ├── dataingestion_6.png ├── architecturemulti.png ├── terraform-folders.png ├── githubfiledownload-1.png ├── architecture-multi-raw.vsdx └── MultiTenantVisualization.PNG ├── Utils ├── appsettings.Development.json ├── appsettings.json ├── scripts │ ├── csv_Import │ │ ├── subscriptions.csv │ │ └── ResourceTypes.csv │ ├── Terraform │ │ ├── resources │ │ │ ├── variables.tf │ │ │ └── post_install.sh │ │ ├── grafana-dashboards │ │ │ ├── variables.tf │ │ │ └── main.tf │ │ └── grafana-datasource │ │ │ ├── variables.tf │ │ │ └── main.tf │ ├── pre-requisites.sh │ ├── grafana-datasource.sh │ ├── update_drilldown.sh │ ├── dashboard_templates │ │ ├── Keyvault-1679088939482.json │ │ ├── Loadbalancer-1679088952762.json │ │ ├── LogAnalytics-1688018903992.json │ │ ├── Eventhubs-1687851669082.json │ │ ├── Firewalls-1689786810784.json │ │ ├── Storage-1679088963314.json │ │ ├── CosmosDB-1679088907885.json │ │ ├── AksServerNode-1679088882867.json │ │ └── ContainerRegistry-1687851648145.json │ └── table_scripts.kql ├── Data │ ├── AzureResource.cs │ ├── Tenant.cs │ └── Message.cs ├── Properties │ └── launchSettings.json ├── ServiceBusHelper.cs ├── Utils.csproj ├── AzureMonitorHelper.cs ├── KeyVaultManager.cs ├── ResourceGraphHelper.cs └── AdxClientHelper.cs ├── AdxIngestFunctionApp ├── Properties │ ├── serviceDependencies.local.json │ ├── serviceDependencies.json │ └── serviceDependencies.AdxIngestFunction-onel03 - Zip Deploy.json ├── host.json ├── AdxIngestFunctionApp.csproj ├── .gitignore └── AdxIngestFunction.cs ├── SchedulePipelineFunctionApp ├── host.json ├── Properties │ ├── serviceDependencies.local.json │ ├── serviceDependencies.json │ └── serviceDependencies.TimerStartPipelineFunction-onel03 - Zip Deploy.json ├── SchedulePipelineFunctionApp.csproj ├── .gitignore └── TimerStartPipelineFunction.cs ├── .github ├── CODE_OF_CONDUCT.md ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── DATAINGESTION.md ├── .gitignore ├── LICENSE.md ├── SECURITY.md ├── Observability.sln ├── CONTRIBUTING.md └── README.md /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Images/DrillDown1.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/DrillDown1.PNG -------------------------------------------------------------------------------- /Images/drilldown2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/drilldown2.png -------------------------------------------------------------------------------- /Images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/architecture.png -------------------------------------------------------------------------------- /Images/mytimetrigger.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/mytimetrigger.png -------------------------------------------------------------------------------- /Images/visualization.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/visualization.png -------------------------------------------------------------------------------- /Images/az-cli-version.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/az-cli-version.png -------------------------------------------------------------------------------- /Images/dataingestion_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/dataingestion_1.png -------------------------------------------------------------------------------- /Images/dataingestion_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/dataingestion_2.png -------------------------------------------------------------------------------- /Images/dataingestion_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/dataingestion_3.png -------------------------------------------------------------------------------- /Images/dataingestion_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/dataingestion_4.png -------------------------------------------------------------------------------- /Images/dataingestion_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/dataingestion_5.png -------------------------------------------------------------------------------- /Images/dataingestion_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/dataingestion_6.png -------------------------------------------------------------------------------- /Images/architecturemulti.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/architecturemulti.png -------------------------------------------------------------------------------- /Images/terraform-folders.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/terraform-folders.png -------------------------------------------------------------------------------- /Images/githubfiledownload-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/githubfiledownload-1.png -------------------------------------------------------------------------------- /Images/architecture-multi-raw.vsdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/architecture-multi-raw.vsdx -------------------------------------------------------------------------------- /Images/MultiTenantVisualization.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/mcsa-observability/HEAD/Images/MultiTenantVisualization.PNG -------------------------------------------------------------------------------- /Utils/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Warning" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /Utils/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information", 5 | "Microsoft.AspNetCore": "Warning" 6 | } 7 | }, 8 | "AllowedHosts": "*" 9 | } 10 | -------------------------------------------------------------------------------- /Utils/scripts/csv_Import/subscriptions.csv: -------------------------------------------------------------------------------- 1 | solution,tenancy,component,tenantId,tenantDomain,subscriptionId,createdAt 2 | Solution 1,multi,Solution 1 - Comp 1,,#@microsoft.onmicrosoft.com,,2020-04-16T13:45:34.953Z -------------------------------------------------------------------------------- /AdxIngestFunctionApp/Properties/serviceDependencies.local.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "appInsights1": { 4 | "type": "appInsights.sdk" 5 | }, 6 | "storage1": { 7 | "type": "storage.emulator", 8 | "connectionId": "AzureWebJobsStorage" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /SchedulePipelineFunctionApp/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | }, 11 | "functionTimeout": "00:10:00" 12 | } -------------------------------------------------------------------------------- /SchedulePipelineFunctionApp/Properties/serviceDependencies.local.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "appInsights1": { 4 | "type": "appInsights.sdk" 5 | }, 6 | "storage1": { 7 | "type": "storage.emulator", 8 | "connectionId": "AzureWebJobsStorage" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /SchedulePipelineFunctionApp/Properties/serviceDependencies.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "appInsights1": { 4 | "type": "appInsights", 5 | "connectionId": "APPINSIGHTS_INSTRUMENTATIONKEY" 6 | }, 7 | "storage1": { 8 | "type": "storage", 9 | "connectionId": "AzureWebJobsStorage" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /AdxIngestFunctionApp/Properties/serviceDependencies.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "storage1": { 4 | "type": "storage", 5 | "connectionId": "AzureWebJobsStorage" 6 | }, 7 | "appInsights1": { 8 | "type": "appInsights", 9 | "connectionId": "APPINSIGHTS_INSTRUMENTATIONKEY" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /Utils/Data/AzureResource.cs: -------------------------------------------------------------------------------- 1 | namespace Observability.Utils.Data 2 | { 3 | public class AzureResource 4 | { 5 | public string ID { get; set; } 6 | public string Name { get; set; } 7 | public string SubscriptionId { get; set; } 8 | public string Location { get; set; } 9 | 10 | public AzureResource() 11 | { 12 | ID = Name = SubscriptionId = Location = ""; 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /Utils/Data/Tenant.cs: -------------------------------------------------------------------------------- 1 | namespace Observability.Utils.Data 2 | { 3 | public class Tenant 4 | { 5 | public string ClientId { get; set; } 6 | public string Tenantid { get; set; } 7 | 8 | public string ClientSecret { get; set; } 9 | 10 | 11 | 12 | public Tenant() 13 | { 14 | ClientId = ""; 15 | Tenantid = ""; 16 | ClientSecret = ""; 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /Utils/scripts/Terraform/resources/variables.tf: -------------------------------------------------------------------------------- 1 | variable "prefix" { 2 | type = string 3 | description = "The prefix that is used to identify resources and tag them" 4 | } 5 | 6 | variable "subscriptionId" { 7 | type = string 8 | description = "The Id of the subscription where the user wants to deploy the resources" 9 | } 10 | 11 | variable "location" { 12 | type = string 13 | description = "The location where the user wants to deploy the resources" 14 | } -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /Utils/scripts/Terraform/grafana-dashboards/variables.tf: -------------------------------------------------------------------------------- 1 | # export TF_VAR_token=$(az grafana api-key create --key `date +%s` --name grafana02-grafana -g grafana02-RG -r editor --time-to-live 4m -o json | jq -r .key) 2 | variable "token" { 3 | type = string 4 | nullable = false 5 | sensitive = true 6 | } 7 | 8 | # export TF_VAR_url=$(az grafana show -g grafana02-RG -n grafana02-grafana -o json | jq -r .properties.endpoint) 9 | variable "url" { 10 | type = string 11 | nullable = false 12 | sensitive = false 13 | } 14 | 15 | #export TF_VAR_prefix=$(terraform output -raw prefix) 16 | variable "prefix" { 17 | type = string 18 | nullable = false 19 | sensitive = false 20 | } -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /Utils/scripts/csv_Import/ResourceTypes.csv: -------------------------------------------------------------------------------- 1 | name,type,resultTableName 2 | loganalytics,microsoft.operationalinsights/workspaces,LogAnalytics_Availability 3 | eventhubs,microsoft.eventhub/namespaces,Eventhubs_Availability 4 | acr,microsoft.containerregistry/registries,Container_Registry_Availability 5 | cosmosdb,microsoft.documentdb/databaseaccounts,Cosmosdb_Availability 6 | aksservernode,microsoft.containerservice/managedclusters,Aksservernode_Availability 7 | keyvault,microsoft.keyvault/vaults,Keyvault_Availability 8 | loadbalancer,microsoft.network/loadbalancers,Loadbalancer_Availability 9 | storage,microsoft.storage/storageaccounts,Storage_Availability 10 | firewall,microsoft.network/azurefirewalls,Firewall_Availability 11 | -------------------------------------------------------------------------------- /DATAINGESTION.md: -------------------------------------------------------------------------------- 1 | - Navigate to the ADX cluster 2 | ![dataingestion 1](Images/dataingestion_1.png) 3 | 4 | - Open the ADX data explorer by right clicking on the table name and selecting "Get data" 5 | ![dataingestion 2](Images/dataingestion_2.png) 6 | 7 | - Choose local file as the data source 8 | ![dataingestion 3](Images/dataingestion_3.png) 9 | 10 | - Upload the locally downloaded ResourceTypes.csv 11 | ![dataingestion 4](Images/dataingestion_4.png) 12 | 13 | - Verify if the schema looks good. If not, click on the three dots in the top right to edit the columns 14 | ![dataingestion 5](Images/dataingestion_5.png) 15 | 16 | - Verify if the ingestion was successful 17 | ![dataingestion 6](Images/dataingestion_6.png) 18 | -------------------------------------------------------------------------------- /AdxIngestFunctionApp/Properties/serviceDependencies.AdxIngestFunction-onel03 - Zip Deploy.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "appInsights1": { 4 | "resourceId": "/subscriptions/[parameters('subscriptionId')]/resourceGroups/[parameters('resourceGroupName')]/providers/microsoft.insights/components/AdxIngestFunction-onel03", 5 | "type": "appInsights.azure", 6 | "connectionId": "APPINSIGHTS_INSTRUMENTATIONKEY" 7 | }, 8 | "storage1": { 9 | "resourceId": "/subscriptions/[parameters('subscriptionId')]/resourceGroups/[parameters('resourceGroupName')]/providers/Microsoft.Storage/storageAccounts/onel03stor", 10 | "type": "storage.azure", 11 | "connectionId": "AzureWebJobsStorage" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /AdxIngestFunctionApp/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0", 3 | "logging": { 4 | "applicationInsights": { 5 | "samplingSettings": { 6 | "isEnabled": true, 7 | "excludedTypes": "Request" 8 | } 9 | } 10 | }, 11 | "extensions": { 12 | "serviceBus": { 13 | "clientRetryOptions": { 14 | "mode": "exponential", 15 | "tryTimeout": "00:01:00", 16 | "delay": "00:00:00.80", 17 | "maxDelay": "00:01:00", 18 | "maxRetries": 3 19 | }, 20 | "prefetchCount": 0, 21 | "autoCompleteMessages": true, 22 | "maxAutoLockRenewalDuration": "00:05:00" 23 | //"maxConcurrentCalls": 16 //Cannot be higher than ingest concurrency of adx 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /SchedulePipelineFunctionApp/Properties/serviceDependencies.TimerStartPipelineFunction-onel03 - Zip Deploy.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "appInsights1": { 4 | "resourceId": "/subscriptions/[parameters('subscriptionId')]/resourceGroups/[parameters('resourceGroupName')]/providers/microsoft.insights/components/TimerStartPipelineFunction-onel03", 5 | "type": "appInsights.azure", 6 | "connectionId": "APPINSIGHTS_INSTRUMENTATIONKEY" 7 | }, 8 | "storage1": { 9 | "resourceId": "/subscriptions/[parameters('subscriptionId')]/resourceGroups/[parameters('resourceGroupName')]/providers/Microsoft.Storage/storageAccounts/onel03stor", 10 | "type": "storage.azure", 11 | "connectionId": "AzureWebJobsStorage" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # This .gitignore file was automatically created by Microsoft(R) Visual Studio. 3 | ################################################################################ 4 | 5 | /.vs 6 | /Helper/bin/Debug/net6.0 7 | /Helper/obj 8 | /EpicFunctionApp/EpicFunction1.cs 9 | /Helper/ServiceBus/ServiceBus.cs 10 | /Utils/bin/Debug/net6.0 11 | /Utils/obj 12 | /ScaletestFunctionApp/bin 13 | /ScaletestFunctionApp/obj 14 | /Helper/bin/Release/net6.0 15 | /Utils/bin/Release/net6.0 16 | /SchedulePipelineFunctionApp/Properties/PublishProfiles 17 | /SchedulePipelineFunctionApp/Properties/ServiceDependencies/TimerStartPipelineFunction-onel03 - Zip Deploy 18 | /AdxIngestFunctionApp/Properties 19 | -------------------------------------------------------------------------------- /Utils/Data/Message.cs: -------------------------------------------------------------------------------- 1 | using Microsoft.Azure.Management.ResourceManager.Fluent.Models; 2 | 3 | namespace Observability.Utils.Data 4 | { 5 | public class Message 6 | { 7 | public string Type { get; set; } 8 | public string SubscriptionID { get; set; } 9 | public string Location { get; set; } 10 | public DateTime From { get; set; } 11 | public DateTime To { get; set; } 12 | public string ResultTable { get; set; } 13 | public List Resources { get; set; } 14 | 15 | public string TenantId {get; set; } 16 | 17 | public Message() 18 | { 19 | SubscriptionID = Location = Type = ResultTable = TenantId = ""; 20 | Resources = new List(); 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /Utils/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:64493", 7 | "sslPort": 0 8 | } 9 | }, 10 | "profiles": { 11 | "Helper": { 12 | "commandName": "Project", 13 | "dotnetRunMessages": true, 14 | "launchBrowser": true, 15 | "applicationUrl": "http://localhost:5175", 16 | "environmentVariables": { 17 | "ASPNETCORE_ENVIRONMENT": "Development" 18 | } 19 | }, 20 | "IIS Express": { 21 | "commandName": "IISExpress", 22 | "launchBrowser": true, 23 | "environmentVariables": { 24 | "ASPNETCORE_ENVIRONMENT": "Development" 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /SchedulePipelineFunctionApp/SchedulePipelineFunctionApp.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | net6.0 4 | v4 5 | Library 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | PreserveNewest 17 | 18 | 19 | PreserveNewest 20 | Never 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /AdxIngestFunctionApp/AdxIngestFunctionApp.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | net6.0 4 | v4 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | PreserveNewest 17 | 18 | 19 | PreserveNewest 20 | Never 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /Utils/ServiceBusHelper.cs: -------------------------------------------------------------------------------- 1 | using Azure.Messaging.ServiceBus; 2 | using System.Threading.Tasks; 3 | 4 | namespace Observability.Utils 5 | { 6 | public class ServiceBusHelper 7 | { 8 | private readonly IConfiguration _config; 9 | 10 | public ServiceBusHelper(IConfiguration config) 11 | { 12 | _config = config; 13 | } 14 | 15 | //public async ServiceBusSender GetSenderAsync() 16 | //{ 17 | // string queueName = _config.GetValue("queueName"); 18 | // string connectionString = _config.GetValue("ServiceBusConnection"); 19 | 20 | // var clientOptions = new ServiceBusClientOptions() 21 | // { 22 | // TransportType = ServiceBusTransportType.AmqpWebSockets 23 | // }; 24 | 25 | // await using ServiceBusClient client = new ServiceBusClient(connectionString); 26 | // var sender = client.CreateSender(queueName); 27 | // return sender; 28 | //} 29 | } 30 | } -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 4 | > Please provide us with the following information: 5 | > --------------------------------------------------------------- 6 | 7 | ### This issue is for a: (mark with an `x`) 8 | ``` 9 | - [ ] bug report -> please search issues before submitting 10 | - [ ] feature request 11 | - [ ] documentation issue or request 12 | - [ ] regression (a behavior that used to work and stopped in a new release) 13 | ``` 14 | 15 | ### Minimal steps to reproduce 16 | > 17 | 18 | ### Any log messages given by the failure 19 | > 20 | 21 | ### Expected/desired behavior 22 | > 23 | 24 | ### OS and Version? 25 | > Windows 7, 8 or 10. Linux (which distribution). macOS (Yosemite? El Capitan? Sierra?) 26 | 27 | ### Versions 28 | > 29 | 30 | ### Mention any other details that might be useful 31 | 32 | > --------------------------------------------------------------- 33 | > Thanks! We'll be in touch soon. 34 | -------------------------------------------------------------------------------- /Utils/Utils.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net6.0 5 | enable 6 | enable 7 | Library 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Purpose 2 | 3 | * ... 4 | 5 | ## Does this introduce a breaking change? 6 | 7 | ``` 8 | [ ] Yes 9 | [ ] No 10 | ``` 11 | 12 | ## Pull Request Type 13 | What kind of change does this Pull Request introduce? 14 | 15 | 16 | ``` 17 | [ ] Bugfix 18 | [ ] Feature 19 | [ ] Code style update (formatting, local variables) 20 | [ ] Refactoring (no functional changes, no api changes) 21 | [ ] Documentation content changes 22 | [ ] Other... Please describe: 23 | ``` 24 | 25 | ## How to Test 26 | * Get the code 27 | 28 | ``` 29 | git clone [repo-address] 30 | cd [repo-name] 31 | git checkout [branch-name] 32 | npm install 33 | ``` 34 | 35 | * Test the code 36 | 37 | ``` 38 | ``` 39 | 40 | ## What to Check 41 | Verify that the following are valid 42 | * ... 43 | 44 | ## Other Information 45 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE -------------------------------------------------------------------------------- /Utils/scripts/Terraform/grafana-datasource/variables.tf: -------------------------------------------------------------------------------- 1 | # export TF_VAR_token=$(az grafana api-key create --key `date +%s` --name grafana02-grafana -g grafana02-RG -r editor --time-to-live 4m -o json | jq -r .key) 2 | variable "token" { 3 | type = string 4 | nullable = false 5 | sensitive = true 6 | } 7 | 8 | # export TF_VAR_url=$(az grafana show -g grafana02-RG -n grafana02-grafana -o json | jq -r .properties.endpoint) 9 | variable "url" { 10 | type = string 11 | nullable = false 12 | sensitive = false 13 | } 14 | 15 | # export TF_VAR_sp_object_id=$(terraform output -raw sp_object_id) 16 | variable "sp_object_id" { 17 | type = string 18 | nullable = false 19 | sensitive = true 20 | } 21 | 22 | #export TF_VAR_cluster_url=$(terraform output -raw cluster_url) 23 | variable "cluster_url" { 24 | type = string 25 | nullable = false 26 | sensitive = false 27 | } 28 | 29 | #export TF_VAR-database_name=$(terraform output -raw database_name) 30 | variable "database_name" { 31 | type = string 32 | nullable = false 33 | sensitive = false 34 | } 35 | 36 | #export TF_VAR_prefix=$(terraform output -raw prefix) 37 | variable "prefix" { 38 | type = string 39 | nullable = false 40 | sensitive = false 41 | } 42 | -------------------------------------------------------------------------------- /Utils/AzureMonitorHelper.cs: -------------------------------------------------------------------------------- 1 | using Azure; 2 | using Azure.Identity; 3 | using Azure.Monitor.Query; 4 | using Azure.Monitor.Query.Models; 5 | using Observability.Utils.Data; 6 | using System.Diagnostics; 7 | //TODO: Is apppsettings.json file in this project needed still? 8 | namespace Observability.Utils 9 | { 10 | public class AzureMonitorHelper 11 | { 12 | MetricsQueryClient metricsClient; 13 | public AzureMonitorHelper() 14 | { 15 | metricsClient = new MetricsQueryClient(new DefaultAzureCredential()); 16 | } 17 | 18 | public async Task RunBatchQueryAsync(Message message) 19 | { 20 | string resourceId = message.Resources[0].ID; 21 | 22 | Response results = await metricsClient.QueryResourceAsync(resourceId, new[] { "Microsoft.OperationalInsights/workspaces" }); 23 | 24 | foreach (var metric in results.Value.Metrics) 25 | { 26 | Debug.WriteLine(metric.Name); 27 | foreach (var element in metric.TimeSeries) 28 | { 29 | Debug.WriteLine("Dimensions: " + string.Join(",", element.Metadata)); 30 | 31 | foreach (var metricValue in element.Values) 32 | { 33 | Debug.WriteLine(metricValue); 34 | } 35 | } 36 | } 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /Utils/scripts/Terraform/grafana-datasource/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | azurerm = { 4 | source = "hashicorp/azurerm" 5 | version = "~> 3.10" 6 | } 7 | grafana = { 8 | source = "grafana/grafana" 9 | version = "1.36.1" 10 | } 11 | } 12 | required_version = ">= 1.1.0" 13 | } 14 | 15 | provider "azurerm" { 16 | features { 17 | resource_group { 18 | prevent_deletion_if_contains_resources = false 19 | } 20 | } 21 | } 22 | 23 | # https://learn.microsoft.com/en-gb/azure/managed-grafana/how-to-api-calls 24 | provider "grafana" { 25 | url = var.url 26 | auth = var.token 27 | } 28 | 29 | locals { 30 | dashboard_templates = "${path.cwd}/../../dashboard_templates" 31 | addperm_1 = "chmod 755 ${path.cwd}/../../grafana-datasource.sh" 32 | addperm_2 = "chmod 755 ${path.cwd}/../../update_drilldown.sh" 33 | create_datasource = "${path.cwd}/../../grafana-datasource.sh ${var.prefix} ${var.cluster_url} ${var.database_name} ${local.dashboard_templates}" 34 | update_drilldowns = "${path.cwd}/../../update_drilldown.sh ${var.prefix} ${local.dashboard_templates}" 35 | } 36 | 37 | //add permission to execute the file 38 | resource "null_resource" "add_perm_1" { 39 | provisioner "local-exec" { 40 | command = local.addperm_1 41 | } 42 | triggers = { 43 | addperm_1 = local.addperm_1 44 | } 45 | } 46 | 47 | //create datasource 48 | resource "null_resource" "datasource_create" { 49 | provisioner "local-exec" { 50 | command = local.create_datasource 51 | } 52 | depends_on = [null_resource.add_perm_1] 53 | triggers = { 54 | grafana_datasource = local.create_datasource 55 | } 56 | } -------------------------------------------------------------------------------- /Utils/scripts/Terraform/resources/post_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Retrieve the outputs and export them as environment variables 4 | export spDisplayName=$(terraform output -raw sp_display_name) 5 | export resourceGroupName=$(terraform output -raw resource_group_name) 6 | export appName=$(terraform output -raw app_name) 7 | export storageAccountId=$(terraform output -raw storage_account_id) # Ensure this output is defined in main.tf 8 | 9 | # Step 1: Retrieve the Service Principal Object ID 10 | spObjectId=$(az ad sp list --display-name "$spDisplayName" --query "[0].id" --output tsv) 11 | 12 | # Ensure the SP Object ID was retrieved 13 | if [ -z "$spObjectId" ]; then 14 | echo "Failed to retrieve Service Principal Object ID." 15 | exit 1 16 | fi 17 | 18 | # Step 2: Set the App Setting in the Function App 19 | az functionapp config appsettings set --name "$appName" --resource-group "$resourceGroupName" --settings "kustoMSIObjectId=$spObjectId" 20 | 21 | # Ensure the app setting was updated 22 | if [ $? -ne 0 ]; then 23 | echo "Failed to update app setting 'kustoMSIObjectId'." 24 | exit 1 25 | fi 26 | 27 | echo "App setting 'kustoMSIObjectId' updated successfully." 28 | 29 | # Step 3: Assign the Storage Blob Data Contributor role to the Service Principal 30 | az role assignment create --assignee "$spObjectId" --role "Storage Blob Data Contributor" --scope "$storageAccountId" 31 | 32 | # Ensure the role assignment was successful 33 | if [ $? -ne 0 ]; then 34 | echo "Failed to assign role 'Storage Blob Data Contributor' to the Service Principal." 35 | exit 1 36 | fi 37 | 38 | echo "Role 'Storage Blob Data Contributor' assigned successfully to the Service Principal." -------------------------------------------------------------------------------- /Utils/scripts/pre-requisites.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sudo apt-get install -y git 3 | wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb 4 | sudo dpkg -i packages-microsoft-prod.deb 5 | rm packages-microsoft-prod.deb 6 | sudo apt-get update -y 7 | sudo apt-get install -y dotnet-sdk-6.0 8 | sudo apt-get install -y zip 9 | sudo apt-get install -y jq 10 | ## Install az cli 11 | # 1.Get packages needed for the install process: 12 | sudo apt-get update -y 13 | sudo apt-get install -y ca-certificates curl apt-transport-https lsb-release gnupg 14 | # 2.Download and install the Microsoft signing key: 15 | sudo mkdir -p /etc/apt/keyrings 16 | curl -sLS https://packages.microsoft.com/keys/microsoft.asc | \ 17 | gpg --dearmor | \ 18 | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null 19 | sudo chmod go+r /etc/apt/keyrings/microsoft.gpg 20 | # 3.Add the Azure CLI software repository: 21 | AZ_REPO=$(lsb_release -cs) 22 | echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $AZ_REPO main" | \ 23 | sudo tee /etc/apt/sources.list.d/azure-cli.list 24 | # 4.Update repository information and install the azure-cli package: 25 | sudo apt-get update -y 26 | sudo apt-get install -y azure-cli 27 | #5. Install Terraform 28 | ## 1.Ensure that your system is up to date and you have installed the gnupg, software-properties-common, and curl packages installed 29 | sudo apt-get update && sudo apt-get install -y gnupg software-properties-common 30 | ## 2.Install the HashiCorp GPG key 31 | wget -O- https://apt.releases.hashicorp.com/gpg | \ 32 | gpg --dearmor | \ 33 | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg 34 | ## 3.Verify the key's fingerprint 35 | gpg --no-default-keyring \ 36 | --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg \ 37 | --fingerprint 38 | ## 4.Add the official HashiCorp repository to your system 39 | echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] \ 40 | https://apt.releases.hashicorp.com $(lsb_release -cs) main" | \ 41 | sudo tee /etc/apt/sources.list.d/hashicorp.list 42 | ## 5.Download the package information from HashiCorp 43 | sudo apt update 44 | ## 6.Install Terraform from the new repository 45 | sudo apt-get install terraform -------------------------------------------------------------------------------- /Utils/scripts/grafana-datasource.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | prefix=$1 4 | cluster_Url=$2 5 | dbName=$3 6 | METRICS_FOLDER_PATH=$4 7 | 8 | echo $prefix 9 | echo $cluster_Url 10 | echo $dbName 11 | echo $METRICS_FOLDER_PATH 12 | 13 | az config set extension.use_dynamic_install=yes_without_prompt 14 | 15 | echo "Managed Grafana: Creating Data Explorer Datasource" 16 | az grafana data-source create -n $prefix-grafana --definition '{ 17 | "name": "Observability Metrics Data Source", 18 | "type": "grafana-azure-data-explorer-datasource", 19 | "typeLogoUrl": "public/plugins/grafana-azure-data-explorer-datasource/img/logo.png", 20 | "access": "proxy", 21 | "url": "api/datasources/proxy/2", 22 | "password": "", 23 | "user": "", 24 | "database": "", 25 | "basicAuth": false, 26 | "isDefault": false, 27 | "jsonData": { 28 | "clusterUrl": "'"$cluster_Url"'", 29 | "dataConsistency": "strongconsistency", 30 | "defaultDatabase": "'"$dbName"'", 31 | "defaultEditorMode": "visual", 32 | "schemaMappings": [], 33 | "azureCredentials": { 34 | "authType": "msi" 35 | } 36 | }, 37 | "readOnly": false 38 | }' 39 | 40 | echo "Managed Grafana: Grab the UID of the Azure Data Explorer data source..." 41 | response=$(az grafana data-source show --data-source "Observability Metrics Data Source" --name $prefix-grafana) 42 | uid=$( jq -r '.uid' <<< "$response" ) 43 | echo $uid 44 | 45 | echo $METRICS_FOLDER_PATH 46 | # Populates the dashboards with the data source UID 47 | function populate_datasource_uid() { 48 | FILE_LIST="$METRICS_FOLDER_PATH/*" 49 | for file in $FILE_LIST 50 | do 51 | echo "Managed Grafana: Updating datasource uid for $file file" 52 | echo "$( jq --arg uid "$uid" '.panels[].datasource |= if (.type=="grafana-azure-data-explorer-datasource") then (.uid=$uid) else . end' $file)" > $file 53 | echo "$( jq --arg uid "$uid" '.panels[].targets[]?.datasource |= if (.type=="grafana-azure-data-explorer-datasource") then (.uid=$uid) else . end' $file)" > $file 54 | echo "$( jq --arg dbName "$dbName" '.panels[].targets[]?.database = $dbName' $file)" > $file 55 | echo "$( jq --arg uid "$uid" '.templating.list[].datasource |= if (.type=="grafana-azure-data-explorer-datasource") then (.uid=$uid) else . end' $file)" > $file 56 | sleep 2 57 | done 58 | } 59 | 60 | populate_datasource_uid $uid $METRICS_FOLDER_PATH -------------------------------------------------------------------------------- /Utils/KeyVaultManager.cs: -------------------------------------------------------------------------------- 1 | 2 | using Azure.Identity; 3 | using Observability.Utils.Data; 4 | using Azure.Security.KeyVault.Secrets; 5 | using Newtonsoft.Json; 6 | 7 | 8 | namespace Observability.Utils 9 | { 10 | //TODO: Make methods asynchronous 11 | public class KeyVaultManager 12 | { 13 | 14 | string TENANT_SECRET_PREFIX = "tenant-"; 15 | SecretClient keyVaultClient; 16 | ILogger log; 17 | 18 | public KeyVaultManager(IConfiguration config, ILogger log) 19 | { 20 | this.log = log; 21 | try 22 | { 23 | string keyVaultName = config.GetValue("keyVaultName"); 24 | log.LogInformation("Reading the KeyVault"); 25 | 26 | var kvUri = "https://" + keyVaultName + ".vault.azure.net"; 27 | 28 | var msiCredential = new ManagedIdentityCredential(config.GetValue("msiclientId")); 29 | 30 | keyVaultClient = new SecretClient(new Uri(kvUri), msiCredential); 31 | 32 | if (keyVaultClient == null) 33 | { 34 | log.LogInformation("KeyVault client is null"); 35 | throw new ArgumentNullException($"Please check the keyVaultName"); 36 | } 37 | 38 | } 39 | catch (Exception e) 40 | { 41 | log.LogInformation("Exception failed to create a keyvault"); 42 | log.LogError(e.Message); 43 | throw new Exception($"Message {e.Message} failed to get the keyVault"); 44 | } 45 | } 46 | 47 | public Tenant GetServicePrincipalCredential(string tenantId) 48 | { 49 | Tenant tenantObj = new Tenant(); 50 | var keyName = TENANT_SECRET_PREFIX + tenantId; 51 | 52 | var secret = keyVaultClient.GetSecret(keyName).Value; 53 | KeyVaultSecret keyValueSecret = keyVaultClient.GetSecret(keyName); 54 | 55 | log.LogInformation("Below is the keyvault value"); 56 | log.LogInformation(keyValueSecret.Value); 57 | 58 | string keyValueSecretStr = keyValueSecret.Value; 59 | if (keyValueSecretStr == null) 60 | { 61 | log.LogInformation("Please Add service principal values for tenantId"); 62 | throw new ArgumentNullException($"Secret not found in the keyvault"); 63 | } 64 | tenantObj = System.Text.Json.JsonSerializer.Deserialize(keyValueSecretStr); 65 | 66 | return tenantObj; 67 | } 68 | 69 | 70 | } 71 | } -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet) and [Xamarin](https://github.com/xamarin). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/security.md/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/security.md/msrc/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/security.md/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/security.md/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /Observability.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 17 4 | VisualStudioVersion = 17.4.33213.308 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AdxIngestFunctionApp", "AdxIngestFunctionApp\AdxIngestFunctionApp.csproj", "{B7ABEAF5-D880-423B-A858-DA1C6CEE268B}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SchedulePipelineFunctionApp", "SchedulePipelineFunctionApp\SchedulePipelineFunctionApp.csproj", "{84878825-0C54-466F-84F7-AC6404132E2C}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Utils\Utils.csproj", "{34829A36-B3C3-4377-B527-F35A73F38BCC}" 11 | EndProject 12 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Readme", "Readme", "{36C96AA5-9ACA-4A5B-A0B1-AA0C39EF5DC8}" 13 | ProjectSection(SolutionItems) = preProject 14 | CHANGELOG.MD = CHANGELOG.MD 15 | DATAINGESTION.md = DATAINGESTION.md 16 | LICENSE.MD = LICENSE.MD 17 | README.MD = README.MD 18 | EndProjectSection 19 | EndProject 20 | Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Docs", "Docs", "{70C109EC-63F0-4F3A-870D-8FB893BD14AE}" 21 | ProjectSection(SolutionItems) = preProject 22 | Docs\architecture-raw.vsdx = Docs\architecture-raw.vsdx 23 | Docs\dataingestion-1.png = Docs\dataingestion-1.png 24 | Docs\dataingestion-2.png = Docs\dataingestion-2.png 25 | Docs\dataingestion-3.png = Docs\dataingestion-3.png 26 | Docs\dataingestion-4.png = Docs\dataingestion-4.png 27 | Docs\dataingestion-5.png = Docs\dataingestion-5.png 28 | Docs\dataingestion-6.png = Docs\dataingestion-6.png 29 | Docs\dataingestion-7.png = Docs\dataingestion-7.png 30 | Docs\githubfiledownload-1.png = Docs\githubfiledownload-1.png 31 | Docs\visualization.png = Docs\visualization.png 32 | EndProjectSection 33 | EndProject 34 | Global 35 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 36 | Debug|Any CPU = Debug|Any CPU 37 | Release|Any CPU = Release|Any CPU 38 | EndGlobalSection 39 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 40 | {B7ABEAF5-D880-423B-A858-DA1C6CEE268B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 41 | {B7ABEAF5-D880-423B-A858-DA1C6CEE268B}.Debug|Any CPU.Build.0 = Debug|Any CPU 42 | {B7ABEAF5-D880-423B-A858-DA1C6CEE268B}.Release|Any CPU.ActiveCfg = Release|Any CPU 43 | {B7ABEAF5-D880-423B-A858-DA1C6CEE268B}.Release|Any CPU.Build.0 = Release|Any CPU 44 | {84878825-0C54-466F-84F7-AC6404132E2C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 45 | {84878825-0C54-466F-84F7-AC6404132E2C}.Debug|Any CPU.Build.0 = Debug|Any CPU 46 | {84878825-0C54-466F-84F7-AC6404132E2C}.Release|Any CPU.ActiveCfg = Release|Any CPU 47 | {84878825-0C54-466F-84F7-AC6404132E2C}.Release|Any CPU.Build.0 = Release|Any CPU 48 | {34829A36-B3C3-4377-B527-F35A73F38BCC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 49 | {34829A36-B3C3-4377-B527-F35A73F38BCC}.Debug|Any CPU.Build.0 = Debug|Any CPU 50 | {34829A36-B3C3-4377-B527-F35A73F38BCC}.Release|Any CPU.ActiveCfg = Release|Any CPU 51 | {34829A36-B3C3-4377-B527-F35A73F38BCC}.Release|Any CPU.Build.0 = Release|Any CPU 52 | EndGlobalSection 53 | GlobalSection(SolutionProperties) = preSolution 54 | HideSolutionNode = FALSE 55 | EndGlobalSection 56 | GlobalSection(ExtensibilityGlobals) = postSolution 57 | SolutionGuid = {803B7ABC-EBC4-45D7-8FFD-5B85138EE35F} 58 | EndGlobalSection 59 | EndGlobal 60 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to [project-title] 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 6 | 7 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | 11 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 12 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 13 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 14 | 15 | - [Code of Conduct](#coc) 16 | - [Issues and Bugs](#issue) 17 | - [Feature Requests](#feature) 18 | - [Submission Guidelines](#submit) 19 | 20 | ## Code of Conduct 21 | Help us keep this project open and inclusive. Please read and follow our [Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 22 | 23 | ## Found an Issue? 24 | If you find a bug in the source code or a mistake in the documentation, you can help us by 25 | [submitting an issue](#submit-issue) to the GitHub Repository. Even better, you can 26 | [submit a Pull Request](#submit-pr) with a fix. 27 | 28 | ## Want a Feature? 29 | You can *request* a new feature by [submitting an issue](#submit-issue) to the GitHub 30 | Repository. If you would like to *implement* a new feature, please submit an issue with 31 | a proposal for your work first, to be sure that we can use it. 32 | 33 | * **Small Features** can be crafted and directly [submitted as a Pull Request](#submit-pr). 34 | 35 | ## Submission Guidelines 36 | 37 | ### Submitting an Issue 38 | Before you submit an issue, search the archive, maybe your question was already answered. 39 | 40 | If your issue appears to be a bug, and hasn't been reported, open a new issue. 41 | Help us to maximize the effort we can spend fixing issues and adding new 42 | features, by not reporting duplicate issues. Providing the following information will increase the 43 | chances of your issue being dealt with quickly: 44 | 45 | * **Overview of the Issue** - if an error is being thrown a non-minified stack trace helps 46 | * **Version** - what version is affected (e.g. 0.1.2) 47 | * **Motivation for or Use Case** - explain what are you trying to do and why the current behavior is a bug for you 48 | * **Browsers and Operating System** - is this a problem with all browsers? 49 | * **Reproduce the Error** - provide a live example or a unambiguous set of steps 50 | * **Related Issues** - has a similar issue been reported before? 51 | * **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be 52 | causing the problem (line of code or commit) 53 | 54 | You can file new issues by providing the above information at the corresponding repository's issues link: https://github.com/[organization-name]/[repository-name]/issues/new]. 55 | 56 | ### Submitting a Pull Request (PR) 57 | Before you submit your Pull Request (PR) consider the following guidelines: 58 | 59 | * Search the repository (https://github.com/[organization-name]/[repository-name]/pulls) for an open or closed PR 60 | that relates to your submission. You don't want to duplicate effort. 61 | 62 | * Make your changes in a new git fork: 63 | 64 | * Commit your changes using a descriptive commit message 65 | * Push your fork to GitHub: 66 | * In GitHub, create a pull request 67 | * If we suggest changes then: 68 | * Make the required updates. 69 | * Rebase your fork and force push to your GitHub repository (this will update your Pull Request): 70 | 71 | ```shell 72 | git rebase main -i 73 | git push -f 74 | ``` 75 | 76 | That's it! Thank you for your contribution! 77 | -------------------------------------------------------------------------------- /Utils/scripts/Terraform/grafana-dashboards/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | azurerm = { 4 | source = "hashicorp/azurerm" 5 | version = "~> 3.10" 6 | } 7 | grafana = { 8 | source = "grafana/grafana" 9 | version = "1.36.1" 10 | } 11 | } 12 | required_version = ">= 1.1.0" 13 | } 14 | 15 | provider "azurerm" { 16 | features { 17 | resource_group { 18 | prevent_deletion_if_contains_resources = false 19 | } 20 | } 21 | } 22 | 23 | # https://learn.microsoft.com/en-gb/azure/managed-grafana/how-to-api-calls 24 | provider "grafana" { 25 | url = var.url 26 | auth = var.token 27 | } 28 | 29 | locals { 30 | dashboard_templates = "${path.cwd}/../../dashboard_templates" 31 | addperm_2 = "chmod 755 ${path.cwd}/../../update_drilldown.sh" 32 | update_drilldowns = "${path.cwd}/../../update_drilldown.sh ${var.prefix} ${local.dashboard_templates}" 33 | } 34 | 35 | resource "grafana_folder" "observability" { 36 | title = "Observability_Dashboard" 37 | } 38 | 39 | resource "grafana_dashboard" "resource_observability" { 40 | folder = grafana_folder.observability.id 41 | overwrite = true 42 | config_json = file("../../dashboard_templates/AzureResourceObservability-1687853750785.json") 43 | } 44 | 45 | resource "grafana_dashboard" "aks_server_node" { 46 | folder = grafana_folder.observability.id 47 | overwrite = true 48 | config_json = file("../../dashboard_templates/AksServerNode-1679088882867.json") 49 | depends_on = [grafana_dashboard.resource_observability] 50 | } 51 | 52 | resource "grafana_dashboard" "cosmos_db" { 53 | folder = grafana_folder.observability.id 54 | overwrite = true 55 | config_json = file("../../dashboard_templates/CosmosDB-1679088907885.json") 56 | depends_on = [grafana_dashboard.resource_observability] 57 | } 58 | 59 | resource "grafana_dashboard" "firewalls" { 60 | folder = grafana_folder.observability.id 61 | overwrite = true 62 | config_json = file("../../dashboard_templates/Firewalls-1689786810784.json") 63 | depends_on = [grafana_dashboard.resource_observability] 64 | } 65 | 66 | resource "grafana_dashboard" "keyvault" { 67 | folder = grafana_folder.observability.id 68 | overwrite = true 69 | config_json = file("../../dashboard_templates/Keyvault-1679088939482.json") 70 | depends_on = [grafana_dashboard.resource_observability] 71 | } 72 | 73 | resource "grafana_dashboard" "loadbalancer" { 74 | folder = grafana_folder.observability.id 75 | overwrite = true 76 | config_json = file("../../dashboard_templates/Loadbalancer-1679088952762.json") 77 | depends_on = [grafana_dashboard.resource_observability] 78 | } 79 | 80 | resource "grafana_dashboard" "storage" { 81 | folder = grafana_folder.observability.id 82 | overwrite = true 83 | config_json = file("../../dashboard_templates/Storage-1679088963314.json") 84 | depends_on = [grafana_dashboard.resource_observability] 85 | } 86 | 87 | resource "grafana_dashboard" "eventhubs" { 88 | folder = grafana_folder.observability.id 89 | overwrite = true 90 | config_json = file("../../dashboard_templates/Eventhubs-1687851669082.json") 91 | depends_on = [grafana_dashboard.resource_observability] 92 | } 93 | 94 | resource "grafana_dashboard" "containerregistry" { 95 | folder = grafana_folder.observability.id 96 | overwrite = true 97 | config_json = file("../../dashboard_templates/ContainerRegistry-1687851648145.json") 98 | depends_on = [grafana_dashboard.resource_observability] 99 | } 100 | 101 | resource "grafana_dashboard" "loganalytics" { 102 | folder = grafana_folder.observability.id 103 | overwrite = true 104 | config_json = file("../../dashboard_templates/LogAnalytics-1688018903992.json") 105 | depends_on = [grafana_dashboard.resource_observability] 106 | } 107 | 108 | //add permission to execute the file 109 | resource "null_resource" "add_perm_2" { 110 | provisioner "local-exec" { 111 | command = local.addperm_2 112 | } 113 | triggers = { 114 | addperm2 = local.addperm_2 115 | } 116 | depends_on = [grafana_dashboard.storage,grafana_dashboard.loadbalancer,grafana_dashboard.keyvault,grafana_dashboard.firewalls,grafana_dashboard.cosmos_db,grafana_dashboard.aks_server_node,grafana_dashboard.resource_observability,grafana_dashboard.eventhubs,grafana_dashboard.containerregistry] 117 | } 118 | 119 | //update uid of datasource on the dashboards 120 | resource "null_resource" "update_dashboard_datasourceuid" { 121 | provisioner "local-exec" { 122 | command = local.update_drilldowns 123 | } 124 | depends_on = [null_resource.add_perm_2] 125 | triggers = { 126 | datasourceuid_update = local.update_drilldowns 127 | } 128 | } -------------------------------------------------------------------------------- /Utils/ResourceGraphHelper.cs: -------------------------------------------------------------------------------- 1 | 2 | using Azure.Identity; 3 | using Azure.ResourceManager; 4 | using Azure.ResourceManager.ResourceGraph; 5 | using Azure.ResourceManager.ResourceGraph.Models; 6 | using Observability.Utils.Data; 7 | using Microsoft.Azure.Management.ResourceGraph.Models; 8 | using Newtonsoft.Json; 9 | using System.Text; 10 | using Azure.Security.KeyVault.Secrets; 11 | 12 | 13 | namespace Observability.Utils 14 | { 15 | //TODO: Make methods asynchronous 16 | public class ResourceGraphHelper 17 | { 18 | ArmClient client; 19 | 20 | Tenant tenantObj; 21 | 22 | private static KeyVaultManager keyVaultManager = null; 23 | 24 | public ResourceGraphHelper(IConfiguration config, ILogger log, string tenantId) 25 | { 26 | 27 | string msftTenantId = config.GetValue("msftTenantId"); 28 | 29 | if (tenantId != null && tenantId == msftTenantId) { 30 | client = new ArmClient( 31 | new ManagedIdentityCredential(config.GetValue("msiclientId"))); 32 | return; 33 | } 34 | 35 | try{ 36 | log.LogInformation($"Creating Arm Client for {tenantId}"); 37 | 38 | string clientId = ""; 39 | string clientSecret = ""; 40 | 41 | //client = new ArmClient( 42 | // new ManagedIdentityCredential(config.GetValue("msiclientId"))); 43 | // Commentted for MultiTenant changes 44 | 45 | //var tenant = client.GetTenants().FirstOrDefault(); 46 | if(tenantId == null) 47 | { 48 | log.LogInformation($"Error Something went wrong TenantId is null"); 49 | throw new Exception($"Message failed to get the TenantId"); 50 | } 51 | log.LogInformation($"Reading the KeyVault for tenantId {tenantId}"); 52 | if(keyVaultManager == null) 53 | { 54 | keyVaultManager = new KeyVaultManager(config, log); 55 | } 56 | 57 | tenantObj = keyVaultManager.GetServicePrincipalCredential(tenantId); 58 | 59 | 60 | if(tenantObj == null) { 61 | log.LogInformation($"SP not defined in keyvault"); 62 | throw new Exception($"Message failed to get the SP credential"); 63 | } 64 | 65 | clientId = tenantObj.ClientId; 66 | clientSecret = tenantObj.ClientSecret; 67 | var credential = new ClientSecretCredential(tenantId, clientId, clientSecret); 68 | client = new ArmClient( 69 | credential); 70 | 71 | log.LogInformation("Created new Arm Client successfully"); 72 | 73 | } 74 | catch(Exception e) 75 | { 76 | log.LogInformation("Exception failed to read secret from keyvault"); 77 | log.LogError(e.Message); 78 | throw new Exception($"Message {e.Message} failed to get the keyVault"); 79 | } 80 | } 81 | 82 | public ResourceQueryResult QueryGraph(string subscriptionId, string resourceType) 83 | { 84 | var tenant = client.GetTenants().FirstOrDefault(); 85 | 86 | string query = $"Resources | where subscriptionId == '{subscriptionId}' | where type == '{resourceType}' | distinct id, name, subscriptionId, location | sort by location asc"; 87 | 88 | var request = new QueryRequest(query); 89 | 90 | var queryContent = new ResourceQueryContent(query); 91 | 92 | var response = tenant.GetResources(queryContent); 93 | 94 | var result = response.Value; 95 | 96 | var resources = new List(); 97 | 98 | return result; 99 | } 100 | 101 | public string GetSubscriptionName(string subscriptionId) 102 | { 103 | var tenant = client.GetTenants().FirstOrDefault(); 104 | string query = $"resourcecontainers | where id == \"/subscriptions/{subscriptionId}\" | project name"; 105 | 106 | var request = new QueryRequest(query); 107 | 108 | var queryContent = new ResourceQueryContent(query); 109 | 110 | var response = tenant.GetResources(queryContent); 111 | var result = response.Value.Data; 112 | 113 | var stringSub = Encoding.ASCII.GetString(result); 114 | 115 | List results = JsonConvert.DeserializeObject>(stringSub); 116 | 117 | var subscriptionName = results[0].name; 118 | 119 | return subscriptionName; 120 | } 121 | } 122 | } -------------------------------------------------------------------------------- /Utils/scripts/update_drilldown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | prefix=$1 4 | METRICS_FOLDER_PATH=$2 5 | ## update drill down links 6 | echo "Update drill down links" 7 | endpoint=$(az grafana show --name $prefix-grafana --resource-group $prefix-RG -o tsv --query properties.endpoint) 8 | echo $endpoint 9 | queryparams="?orgId=1\${__url_time_range}&var-selecteddate=\${__data.fields.date}&\${Region:queryparam}&\${Subscriptions:queryparam}&\${Solution:queryparam}" 10 | 11 | storageuid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'Storage')].uid | [1]" -o tsv) 12 | storagedrilldown=$endpoint/d/$storageuid/storage$queryparams 13 | echo $storagedrilldown 14 | 15 | 16 | keyvaultuid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'Keyvault')].uid" -o tsv) 17 | keyvaultdrilldown=$endpoint/d/$keyvaultuid/keyvault$queryparams 18 | echo $keyvaultdrilldown 19 | 20 | aksuid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'AksServerNode')].uid" -o tsv) 21 | aksdrilldown=$endpoint/d/$aksuid/aksservernode$queryparams 22 | echo $aksdrilldown 23 | 24 | firewalluid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'Firewalls')].uid" -o tsv) 25 | firewalldrilldown=$endpoint/d/$firewalluid/firewalls$queryparams 26 | echo $firewalldrilldown 27 | 28 | lbuid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'Loadbalancer')].uid" -o tsv) 29 | lbdrilldown=$endpoint/d/$lbuid/loadbalancer$queryparams 30 | echo $lbdrilldown 31 | 32 | cosmosdbuid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'CosmosDB')].uid" -o tsv) 33 | cosmosdbdrilldown=$endpoint/d/$cosmosdbuid/cosmosdb-details$queryparams 34 | echo $cosmosdbdrilldown 35 | 36 | cognitiveservicebuid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'CognitiveServices')].uid" -o tsv) 37 | cognitiveservicedrilldown=$endpoint/d/$cognitiveservicebuid/cognitiveservices$queryparams 38 | echo $cognitiveservicedrilldown 39 | 40 | containerregistrybuid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'ContainerRegistry')].uid" -o tsv) 41 | containerregistrydrilldown=$endpoint/d/$containerregistrybuid/containerregistry$queryparams 42 | echo $containerregistrydown 43 | 44 | eventhubsbuid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'Eventhubs')].uid" -o tsv) 45 | eventhubsdrilldown=$endpoint/d/$eventhubsbuid/eventhubs$queryparams 46 | echo $eventhubsdrilldown 47 | 48 | loganalyticsbuid=$(az grafana dashboard list --name $prefix-grafana --resource-group $prefix-RG --query "[?contains(@.title, 'LogAnalytics')].uid" -o tsv) 49 | loganalyticsdrilldown=$endpoint/d/$loganalyticsbuid/loganalytics$queryparams 50 | echo $loganalyticsdrilldown 51 | 52 | jsonfile=$METRICS_FOLDER_PATH/AzureResourceObservability-1687853750785.json 53 | echo $jsonfile 54 | 55 | echo "$(jq --arg storagedrilldown "$storagedrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="storage drill down details") then .url=$storagedrilldown else . end' $jsonfile)" > $jsonfile 56 | 57 | echo "$(jq --arg keyvaultdrilldown "$keyvaultdrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="keyvault drill down details") then .url=$keyvaultdrilldown else . end' $jsonfile)" > $jsonfile 58 | 59 | echo "$(jq --arg aksdrilldown "$aksdrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="aksservernode drill down details") then .url=$aksdrilldown else . end' $jsonfile)" > $jsonfile 60 | 61 | echo "$(jq --arg firewalldrilldown "$firewalldrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="firewall drill down details") then .url=$firewalldrilldown else . end' $jsonfile)" > $jsonfile 62 | 63 | echo "$(jq --arg lbdrilldown "$lbdrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="loadbalancer drill down details") then .url=$lbdrilldown else . end' $jsonfile)" > $jsonfile 64 | 65 | echo "$(jq --arg cosmosdbdrilldown "$cosmosdbdrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="cosmosdb drill down details") then .url=$cosmosdbdrilldown else . end' $jsonfile)" > $jsonfile 66 | 67 | echo "$(jq --arg cognitiveservicedrilldown "$cognitiveservicedrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="cognitive service drill down details") then .url=$cognitiveservicedrilldown else . end' $jsonfile)" > $jsonfile 68 | 69 | echo "$(jq --arg containerregistrydrilldown "$containerregistrydrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="acr drill down details") then .url=$containerregistrydrilldown else . end' $jsonfile)" > $jsonfile 70 | 71 | echo "$(jq --arg eventhubsdrilldown "$eventhubsdrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="eventhubs drill down details") then .url=$eventhubsdrilldown else . end' $jsonfile)" > $jsonfile 72 | 73 | echo "$(jq --arg loganalyticsdrilldown "$loganalyticsdrilldown" '.panels[].fieldConfig.defaults.links[]? |= if(.title=="loganalytics drill down details") then .url=$loganalyticsdrilldown else . end' $jsonfile)" > $jsonfile 74 | 75 | 76 | echo "Managed Grafana: Importing dashboard for $jsonfile file" 77 | az grafana dashboard update -g $prefix-RG -n $prefix-grafana --folder Observability_Dashboard --overwrite true --definition @$jsonfile 78 | sleep 2 -------------------------------------------------------------------------------- /AdxIngestFunctionApp/.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Azure Functions localsettings file 5 | local.settings.json 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | bld/ 24 | [Bb]in/ 25 | [Oo]bj/ 26 | [Ll]og/ 27 | 28 | # Visual Studio 2015 cache/options directory 29 | .vs/ 30 | # Uncomment if you have tasks that create the project's static files in wwwroot 31 | #wwwroot/ 32 | 33 | # MSTest test Results 34 | [Tt]est[Rr]esult*/ 35 | [Bb]uild[Ll]og.* 36 | 37 | # NUNIT 38 | *.VisualState.xml 39 | TestResult.xml 40 | 41 | # Build Results of an ATL Project 42 | [Dd]ebugPS/ 43 | [Rr]eleasePS/ 44 | dlldata.c 45 | 46 | # DNX 47 | project.lock.json 48 | project.fragment.lock.json 49 | artifacts/ 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # NCrunch 117 | _NCrunch_* 118 | .*crunch*.local.xml 119 | nCrunchTemp_* 120 | 121 | # MightyMoose 122 | *.mm.* 123 | AutoTest.Net/ 124 | 125 | # Web workbench (sass) 126 | .sass-cache/ 127 | 128 | # Installshield output folder 129 | [Ee]xpress/ 130 | 131 | # DocProject is a documentation generator add-in 132 | DocProject/buildhelp/ 133 | DocProject/Help/*.HxT 134 | DocProject/Help/*.HxC 135 | DocProject/Help/*.hhc 136 | DocProject/Help/*.hhk 137 | DocProject/Help/*.hhp 138 | DocProject/Help/Html2 139 | DocProject/Help/html 140 | 141 | # Click-Once directory 142 | publish/ 143 | 144 | # Publish Web Output 145 | *.[Pp]ublish.xml 146 | *.azurePubxml 147 | # TODO: Comment the next line if you want to checkin your web deploy settings 148 | # but database connection strings (with potential passwords) will be unencrypted 149 | #*.pubxml 150 | *.publishproj 151 | 152 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 153 | # checkin your Azure Web App publish settings, but sensitive information contained 154 | # in these scripts will be unencrypted 155 | PublishScripts/ 156 | 157 | # NuGet Packages 158 | *.nupkg 159 | # The packages folder can be ignored because of Package Restore 160 | **/packages/* 161 | # except build/, which is used as an MSBuild target. 162 | !**/packages/build/ 163 | # Uncomment if necessary however generally it will be regenerated when needed 164 | #!**/packages/repositories.config 165 | # NuGet v3's project.json files produces more ignoreable files 166 | *.nuget.props 167 | *.nuget.targets 168 | 169 | # Microsoft Azure Build Output 170 | csx/ 171 | *.build.csdef 172 | 173 | # Microsoft Azure Emulator 174 | ecf/ 175 | rcf/ 176 | 177 | # Windows Store app package directories and files 178 | AppPackages/ 179 | BundleArtifacts/ 180 | Package.StoreAssociation.xml 181 | _pkginfo.txt 182 | 183 | # Visual Studio cache files 184 | # files ending in .cache can be ignored 185 | *.[Cc]ache 186 | # but keep track of directories ending in .cache 187 | !*.[Cc]ache/ 188 | 189 | # Others 190 | ClientBin/ 191 | ~$* 192 | *~ 193 | *.dbmdl 194 | *.dbproj.schemaview 195 | *.jfm 196 | *.pfx 197 | *.publishsettings 198 | node_modules/ 199 | orleans.codegen.cs 200 | 201 | # Since there are multiple workflows, uncomment next line to ignore bower_components 202 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 203 | #bower_components/ 204 | 205 | # RIA/Silverlight projects 206 | Generated_Code/ 207 | 208 | # Backup & report files from converting an old project file 209 | # to a newer Visual Studio version. Backup files are not needed, 210 | # because we have git ;-) 211 | _UpgradeReport_Files/ 212 | Backup*/ 213 | UpgradeLog*.XML 214 | UpgradeLog*.htm 215 | 216 | # SQL Server files 217 | *.mdf 218 | *.ldf 219 | 220 | # Business Intelligence projects 221 | *.rdl.data 222 | *.bim.layout 223 | *.bim_*.settings 224 | 225 | # Microsoft Fakes 226 | FakesAssemblies/ 227 | 228 | # GhostDoc plugin setting file 229 | *.GhostDoc.xml 230 | 231 | # Node.js Tools for Visual Studio 232 | .ntvs_analysis.dat 233 | 234 | # Visual Studio 6 build log 235 | *.plg 236 | 237 | # Visual Studio 6 workspace options file 238 | *.opt 239 | 240 | # Visual Studio LightSwitch build output 241 | **/*.HTMLClient/GeneratedArtifacts 242 | **/*.DesktopClient/GeneratedArtifacts 243 | **/*.DesktopClient/ModelManifest.xml 244 | **/*.Server/GeneratedArtifacts 245 | **/*.Server/ModelManifest.xml 246 | _Pvt_Extensions 247 | 248 | # Paket dependency manager 249 | .paket/paket.exe 250 | paket-files/ 251 | 252 | # FAKE - F# Make 253 | .fake/ 254 | 255 | # JetBrains Rider 256 | .idea/ 257 | *.sln.iml 258 | 259 | # CodeRush 260 | .cr/ 261 | 262 | # Python Tools for Visual Studio (PTVS) 263 | __pycache__/ 264 | *.pyc -------------------------------------------------------------------------------- /SchedulePipelineFunctionApp/.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # Azure Functions localsettings file 5 | local.settings.json 6 | 7 | # User-specific files 8 | *.suo 9 | *.user 10 | *.userosscache 11 | *.sln.docstates 12 | 13 | # User-specific files (MonoDevelop/Xamarin Studio) 14 | *.userprefs 15 | 16 | # Build results 17 | [Dd]ebug/ 18 | [Dd]ebugPublic/ 19 | [Rr]elease/ 20 | [Rr]eleases/ 21 | x64/ 22 | x86/ 23 | bld/ 24 | [Bb]in/ 25 | [Oo]bj/ 26 | [Ll]og/ 27 | 28 | # Visual Studio 2015 cache/options directory 29 | .vs/ 30 | # Uncomment if you have tasks that create the project's static files in wwwroot 31 | #wwwroot/ 32 | 33 | # MSTest test Results 34 | [Tt]est[Rr]esult*/ 35 | [Bb]uild[Ll]og.* 36 | 37 | # NUNIT 38 | *.VisualState.xml 39 | TestResult.xml 40 | 41 | # Build Results of an ATL Project 42 | [Dd]ebugPS/ 43 | [Rr]eleasePS/ 44 | dlldata.c 45 | 46 | # DNX 47 | project.lock.json 48 | project.fragment.lock.json 49 | artifacts/ 50 | 51 | *_i.c 52 | *_p.c 53 | *_i.h 54 | *.ilk 55 | *.meta 56 | *.obj 57 | *.pch 58 | *.pdb 59 | *.pgc 60 | *.pgd 61 | *.rsp 62 | *.sbr 63 | *.tlb 64 | *.tli 65 | *.tlh 66 | *.tmp 67 | *.tmp_proj 68 | *.log 69 | *.vspscc 70 | *.vssscc 71 | .builds 72 | *.pidb 73 | *.svclog 74 | *.scc 75 | 76 | # Chutzpah Test files 77 | _Chutzpah* 78 | 79 | # Visual C++ cache files 80 | ipch/ 81 | *.aps 82 | *.ncb 83 | *.opendb 84 | *.opensdf 85 | *.sdf 86 | *.cachefile 87 | *.VC.db 88 | *.VC.VC.opendb 89 | 90 | # Visual Studio profiler 91 | *.psess 92 | *.vsp 93 | *.vspx 94 | *.sap 95 | 96 | # TFS 2012 Local Workspace 97 | $tf/ 98 | 99 | # Guidance Automation Toolkit 100 | *.gpState 101 | 102 | # ReSharper is a .NET coding add-in 103 | _ReSharper*/ 104 | *.[Rr]e[Ss]harper 105 | *.DotSettings.user 106 | 107 | # JustCode is a .NET coding add-in 108 | .JustCode 109 | 110 | # TeamCity is a build add-in 111 | _TeamCity* 112 | 113 | # DotCover is a Code Coverage Tool 114 | *.dotCover 115 | 116 | # NCrunch 117 | _NCrunch_* 118 | .*crunch*.local.xml 119 | nCrunchTemp_* 120 | 121 | # MightyMoose 122 | *.mm.* 123 | AutoTest.Net/ 124 | 125 | # Web workbench (sass) 126 | .sass-cache/ 127 | 128 | # Installshield output folder 129 | [Ee]xpress/ 130 | 131 | # DocProject is a documentation generator add-in 132 | DocProject/buildhelp/ 133 | DocProject/Help/*.HxT 134 | DocProject/Help/*.HxC 135 | DocProject/Help/*.hhc 136 | DocProject/Help/*.hhk 137 | DocProject/Help/*.hhp 138 | DocProject/Help/Html2 139 | DocProject/Help/html 140 | 141 | # Click-Once directory 142 | publish/ 143 | 144 | # Publish Web Output 145 | *.[Pp]ublish.xml 146 | *.azurePubxml 147 | # TODO: Comment the next line if you want to checkin your web deploy settings 148 | # but database connection strings (with potential passwords) will be unencrypted 149 | #*.pubxml 150 | *.publishproj 151 | 152 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 153 | # checkin your Azure Web App publish settings, but sensitive information contained 154 | # in these scripts will be unencrypted 155 | PublishScripts/ 156 | 157 | # NuGet Packages 158 | *.nupkg 159 | # The packages folder can be ignored because of Package Restore 160 | **/packages/* 161 | # except build/, which is used as an MSBuild target. 162 | !**/packages/build/ 163 | # Uncomment if necessary however generally it will be regenerated when needed 164 | #!**/packages/repositories.config 165 | # NuGet v3's project.json files produces more ignoreable files 166 | *.nuget.props 167 | *.nuget.targets 168 | 169 | # Microsoft Azure Build Output 170 | csx/ 171 | *.build.csdef 172 | 173 | # Microsoft Azure Emulator 174 | ecf/ 175 | rcf/ 176 | 177 | # Windows Store app package directories and files 178 | AppPackages/ 179 | BundleArtifacts/ 180 | Package.StoreAssociation.xml 181 | _pkginfo.txt 182 | 183 | # Visual Studio cache files 184 | # files ending in .cache can be ignored 185 | *.[Cc]ache 186 | # but keep track of directories ending in .cache 187 | !*.[Cc]ache/ 188 | 189 | # Others 190 | ClientBin/ 191 | ~$* 192 | *~ 193 | *.dbmdl 194 | *.dbproj.schemaview 195 | *.jfm 196 | *.pfx 197 | *.publishsettings 198 | node_modules/ 199 | orleans.codegen.cs 200 | 201 | # Since there are multiple workflows, uncomment next line to ignore bower_components 202 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 203 | #bower_components/ 204 | 205 | # RIA/Silverlight projects 206 | Generated_Code/ 207 | 208 | # Backup & report files from converting an old project file 209 | # to a newer Visual Studio version. Backup files are not needed, 210 | # because we have git ;-) 211 | _UpgradeReport_Files/ 212 | Backup*/ 213 | UpgradeLog*.XML 214 | UpgradeLog*.htm 215 | 216 | # SQL Server files 217 | *.mdf 218 | *.ldf 219 | 220 | # Business Intelligence projects 221 | *.rdl.data 222 | *.bim.layout 223 | *.bim_*.settings 224 | 225 | # Microsoft Fakes 226 | FakesAssemblies/ 227 | 228 | # GhostDoc plugin setting file 229 | *.GhostDoc.xml 230 | 231 | # Node.js Tools for Visual Studio 232 | .ntvs_analysis.dat 233 | 234 | # Visual Studio 6 build log 235 | *.plg 236 | 237 | # Visual Studio 6 workspace options file 238 | *.opt 239 | 240 | # Visual Studio LightSwitch build output 241 | **/*.HTMLClient/GeneratedArtifacts 242 | **/*.DesktopClient/GeneratedArtifacts 243 | **/*.DesktopClient/ModelManifest.xml 244 | **/*.Server/GeneratedArtifacts 245 | **/*.Server/ModelManifest.xml 246 | _Pvt_Extensions 247 | 248 | # Paket dependency manager 249 | .paket/paket.exe 250 | paket-files/ 251 | 252 | # FAKE - F# Make 253 | .fake/ 254 | 255 | # JetBrains Rider 256 | .idea/ 257 | *.sln.iml 258 | 259 | # CodeRush 260 | .cr/ 261 | 262 | # Python Tools for Visual Studio (PTVS) 263 | __pycache__/ 264 | *.pyc -------------------------------------------------------------------------------- /SchedulePipelineFunctionApp/TimerStartPipelineFunction.cs: -------------------------------------------------------------------------------- 1 | using Azure.Identity; 2 | using Azure.Messaging.ServiceBus; 3 | using Observability.Utils; 4 | using Observability.Utils.Data; 5 | using Kusto.Data.Net.Client; 6 | using Microsoft.Azure.WebJobs; 7 | using Microsoft.Extensions.Configuration; 8 | using Microsoft.Extensions.Logging; 9 | using Newtonsoft.Json; 10 | using System; 11 | using System.Collections.Generic; 12 | using System.Text; 13 | using System.Threading.Tasks; 14 | using System.Net.Http; 15 | using System.Runtime.CompilerServices; 16 | 17 | namespace Observability.SchedulePipelineFunctionApp 18 | { 19 | public class TimerStartPipelineFunction 20 | { 21 | [FunctionName("TimerStartPipelineFunction")] 22 | public static async Task Run([TimerTrigger("%MyTimeTrigger%")] TimerInfo myTimer, ILogger log) 23 | { 24 | log.LogInformation($"TimerStartPipelineFunction started {DateTime.Now}"); 25 | 26 | var config = new ConfigurationBuilder().AddEnvironmentVariables().Build(); 27 | 28 | string queueName = config.GetValue("queueName"); 29 | string ServiceBusNamespace = config.GetValue("serviceBusNameSpace"); 30 | var fullyQualifiedNamespace = $"{ServiceBusNamespace}.servicebus.windows.net"; 31 | 32 | var managedIdentityCredential = new ManagedIdentityCredential(config.GetValue("msiclientId")); 33 | 34 | log.LogInformation($"Service Bus Namespace: {fullyQualifiedNamespace}"); 35 | await using ServiceBusClient client = new ServiceBusClient(fullyQualifiedNamespace, managedIdentityCredential); 36 | log.LogInformation($"Service Bus Client: {client} created"); 37 | var sender = client.CreateSender(queueName); 38 | 39 | //await using var sender = await sbHelper.GetSenderAsync(); // C# 8 feature. 40 | 41 | var adx = new AdxClientHelper(config, log); //TODO Remove hard coding. 42 | 43 | var kcsb = adx.GetClient(); 44 | 45 | using var kustoClient = KustoClientFactory.CreateCslQueryProvider(kcsb); 46 | 47 | var subscriptionsUpdateQuery = @"Subscriptions 48 | | join kind=leftouter Subscriptions_Processed on $left.subscriptionId == $right.subscriptionId 49 | | extend dp = iff(isempty(dateProcessed),ago(1d),dateProcessed) 50 | | summarize dateProcessedThrough = max(dp) by tenantId, subscriptionId 51 | | order by dateProcessedThrough asc, tenantId, subscriptionId"; 52 | 53 | var resourceTypeQuery = @"Resource_Providers"; 54 | 55 | using var reader = kustoClient.ExecuteQuery(subscriptionsUpdateQuery); 56 | 57 | // var resourceClient = new ResourceGraphHelper(config, log); // Commented for multi tenant changes. 58 | 59 | string prevTenantId = ""; 60 | ResourceGraphHelper resourceClient = null; 61 | 62 | while (reader.Read()) 63 | { 64 | using var resourceTypes = kustoClient.ExecuteQuery(resourceTypeQuery); //TODO: I moved here so that you don't need to create again after "while (resourceTypes.Read())" loop. Does that work? 65 | 66 | var tenantId = reader.GetGuid(0).ToString(); 67 | 68 | log.LogInformation($"This is the current Tenant ID: {tenantId}"); 69 | 70 | var subscriptionId = reader.GetGuid(1); 71 | log.LogInformation($"This is the current Subscription ID: {subscriptionId}"); 72 | 73 | var fromDate = reader.GetDateTime(2); 74 | var toDate = DateTime.UtcNow; 75 | 76 | var subscriptionNameQuery = $"Subscription_Names | where subscriptionId == '{subscriptionId}'"; 77 | 78 | var subscriptionNameResponse = kustoClient.ExecuteQuery(subscriptionNameQuery); 79 | 80 | if(tenantId != prevTenantId) { 81 | log.LogInformation($"Creating new resource client for {tenantId}"); 82 | prevTenantId = tenantId; 83 | resourceClient = new ResourceGraphHelper(config, log, tenantId); 84 | } 85 | 86 | var subscriptionName = ""; 87 | if (subscriptionNameResponse.Read()) 88 | { 89 | subscriptionNameResponse.Read(); 90 | subscriptionName = subscriptionNameResponse.GetString(1); 91 | } 92 | else 93 | { 94 | subscriptionName = resourceClient.GetSubscriptionName(subscriptionId.ToString()); //TODO: make asynchronous 95 | await adx.IngestSubscriptionNameAsync(subscriptionId.ToString(), subscriptionName); 96 | } 97 | 98 | log.LogInformation($"This is the Subscription name: {subscriptionName}"); 99 | 100 | while (resourceTypes.Read()) 101 | { 102 | var resources = new List(); 103 | var type = resourceTypes.GetString(1); 104 | var resultTable = resourceTypes.GetString(2); 105 | 106 | var result = resourceClient.QueryGraph(subscriptionId.ToString(), type); 107 | 108 | log.LogInformation($"This is the graph result : {result.Data}"); 109 | 110 | var resultArray = result.Data.ToArray(); 111 | string str = Encoding.ASCII.GetString(resultArray); 112 | 113 | resources = JsonConvert.DeserializeObject>(str); 114 | 115 | var finalResources = new List(); 116 | 117 | // Send to service bus and clear resources array for next batch 118 | using ServiceBusMessageBatch messageBatch = await sender.CreateMessageBatchAsync(); 119 | 120 | int i = 0; 121 | int j = 0; 122 | while (i < resources.Count) 123 | { 124 | var queueMessage = new Message(); 125 | 126 | queueMessage.SubscriptionID = subscriptionId.ToString(); 127 | queueMessage.Type = type; 128 | queueMessage.From = fromDate; 129 | queueMessage.To = toDate; 130 | queueMessage.ResultTable = resultTable; 131 | queueMessage.TenantId = tenantId ; 132 | 133 | var curResource = resources[i]; 134 | var curLocation = curResource.Location; 135 | 136 | var count = 0; 137 | while ((j < resources.Count) && (curLocation == resources[j].Location) && (count < 50)) 138 | { 139 | finalResources.Add(resources[j]); 140 | count++; 141 | j++; 142 | i = j; 143 | } 144 | 145 | queueMessage.Location = curLocation; 146 | queueMessage.Resources = finalResources; 147 | 148 | var messageJson = System.Text.Json.JsonSerializer.Serialize(queueMessage); 149 | 150 | log.LogInformation("Adding message to queue"); 151 | if (messageBatch.TryAddMessage(new ServiceBusMessage(messageJson))) 152 | { 153 | log.LogInformation("Message added to queue"); 154 | 155 | log.LogInformation("Sending messages now..."); 156 | await sender.SendMessagesAsync(messageBatch); 157 | 158 | } 159 | else 160 | { 161 | throw new Exception($"Message {messageJson} failed to get to queue"); 162 | } 163 | 164 | finalResources.Clear(); 165 | } 166 | 167 | log.LogInformation("Clearing resources"); 168 | resources.Clear(); 169 | messageBatch.Dispose(); 170 | } 171 | await adx.IngestSubscriptionDateAsync(subscriptionId.ToString(), toDate); 172 | resourceTypes.Close(); 173 | } 174 | reader.Close(); 175 | await sender.CloseAsync(); 176 | } 177 | } 178 | } -------------------------------------------------------------------------------- /Utils/AdxClientHelper.cs: -------------------------------------------------------------------------------- 1 | using Azure.Storage; 2 | using Azure.Identity; 3 | using Azure.Storage.Blobs; 4 | using Azure.Storage.Blobs.Models; 5 | using Azure.Storage.Blobs.Specialized; 6 | using Kusto.Data; 7 | using Kusto.Data.Common; 8 | using Kusto.Data.Net.Client; 9 | using Kusto.Ingest; 10 | using System.Text; 11 | using static System.Net.WebRequestMethods; 12 | 13 | namespace Observability.Utils 14 | { 15 | public class AdxClientHelper 16 | { 17 | private readonly IConfiguration _config; 18 | private ILogger log; 19 | private readonly string clusterUri; 20 | private readonly string ingestionUri; 21 | private readonly string databaseName; 22 | private readonly string containerName; 23 | private readonly string storageAccountName; 24 | private readonly string msiClientId; 25 | private readonly string kustoMSIObjectId; 26 | private readonly string keyVaultName; 27 | private static HashSet seenFilePrefixes = new HashSet(); 28 | 29 | 30 | public AdxClientHelper(IConfiguration config, ILogger log) 31 | { 32 | this.log = log; 33 | this._config = config; 34 | this.clusterUri = _config.GetValue("adxConnectionString"); // TODO: Confirm if this is ingeston uri or cluster uri 35 | this.ingestionUri = _config.GetValue("adxIngestionURI"); 36 | this.databaseName = _config.GetValue("metricsdbName"); 37 | this.storageAccountName = _config.GetValue("storageAccountName"); 38 | this.containerName = _config.GetValue("rawDataContainerName"); 39 | this.msiClientId = _config.GetValue("msiclientId"); 40 | this.kustoMSIObjectId = _config.GetValue("kustoMSIObjectId"); 41 | this.keyVaultName = _config.GetValue("keyVaultName"); 42 | } 43 | 44 | public KustoConnectionStringBuilder GetClient() 45 | { 46 | //var kcsb = new KustoConnectionStringBuilder(this.clusterUri, this.databaseName).WithAadSystemManagedIdentity(); 47 | var kcsb = new KustoConnectionStringBuilder(clusterUri, databaseName).WithAadUserManagedIdentity(msiClientId); 48 | return kcsb; 49 | } 50 | 51 | public async Task IngestSubscriptionNameAsync(string subscriptionId, string subscriptionName) 52 | { 53 | var kcsb = GetClient(); 54 | var kustoClient = KustoClientFactory.CreateCslAdminProvider(kcsb); //TODO: Why created in every method, could be in class constructor? 55 | 56 | var ingestQuery = $".ingest inline into table Subscription_Names <| {subscriptionId}, {subscriptionName}"; 57 | await kustoClient.ExecuteControlCommandAsync(databaseName, ingestQuery); 58 | } 59 | 60 | public async Task IngestSubscriptionDateAsync(string subscriptionId, DateTime processedTime) 61 | { 62 | var kcsb = GetClient(); 63 | 64 | var kustoClient = KustoClientFactory.CreateCslAdminProvider(kcsb); //TODO: Why created in every method, could be in class constructor? 65 | 66 | var ingestQuery = $".ingest inline into table Subscriptions_Processed <| {subscriptionId},{processedTime}"; 67 | await kustoClient.ExecuteControlCommandAsync(databaseName, ingestQuery); 68 | } 69 | 70 | public async Task IngestToAdxAsync(string batchResponse, string tableName) 71 | { 72 | string fileName = string.Format(@"{0}.json", Guid.NewGuid()); 73 | 74 | await AppendToBlobAsync(batchResponse, fileName); 75 | 76 | var kcsb = GetClient(); 77 | 78 | var kustoClient = KustoClientFactory.CreateCslAdminProvider(kcsb); 79 | 80 | //// Ingest the JSON data multijson format (meaning one large JSON object and not an object per line) 81 | var ingestCommand = $".ingest into table {tableName}_Raw('https://{storageAccountName}.blob.core.windows.net/{containerName}/{fileName}') with '{{\"format\":\"multijson\", \"ingestionMappingReference\":\"RawMetricsMapping\"}}'"; 82 | await kustoClient.ExecuteControlCommandAsync(databaseName, ingestCommand); 83 | } 84 | 85 | private async Task AppendToBlobAsync(string jsonData, string filePrefix) 86 | { 87 | 88 | if (seenFilePrefixes.Contains(filePrefix)) 89 | { 90 | log.LogInformation($"File prefix {filePrefix} has already been uploaded. Skipping upload to storage container."); 91 | return; 92 | } 93 | 94 | seenFilePrefixes.Add(filePrefix); 95 | log.LogInformation($"Adding file prefix {filePrefix} to seen prefixes"); 96 | 97 | string fileName = string.Format(@"{0}.json", filePrefix); 98 | 99 | // Create a ManagedIdentityCredential object 100 | var managedIdentityCredential = new ManagedIdentityCredential(msiClientId); 101 | // Create a BlobServiceClient object which is used to create a container client 102 | var blobServiceEndpoint = $"https://{storageAccountName}.blob.core.windows.net/"; 103 | BlobServiceClient blobServiceClient = new BlobServiceClient(new Uri(blobServiceEndpoint), managedIdentityCredential); 104 | 105 | BlobContainerClient containerClient = null; 106 | bool bContainerExists = false; 107 | 108 | // Create the container and return a container client object 109 | foreach (BlobContainerItem blobContainerItem in blobServiceClient.GetBlobContainers()) 110 | { 111 | if (blobContainerItem.Name == containerName) 112 | { 113 | bContainerExists = true; 114 | break; 115 | } 116 | } 117 | 118 | // Create or use existing Azure container as client. 119 | if (!bContainerExists) 120 | { 121 | containerClient = blobServiceClient.CreateBlobContainer(containerName); 122 | } 123 | else 124 | containerClient = blobServiceClient.GetBlobContainerClient(containerName); 125 | 126 | BlobClient blobClient = containerClient.GetBlobClient(fileName); 127 | 128 | try 129 | { 130 | // Try to upload the blob only if it doesn't already exist 131 | var stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData)); 132 | stream.Position = 0; 133 | await blobClient.UploadAsync(stream, new BlobUploadOptions { Conditions = new BlobRequestConditions { IfNoneMatch = Azure.ETag.All } }); 134 | } 135 | catch (Azure.RequestFailedException ex) when (ex.ErrorCode == BlobErrorCode.BlobAlreadyExists) 136 | { 137 | log.LogInformation($"Blob {fileName} already exists in the container {containerName}. Skipping upload."); 138 | } 139 | } 140 | 141 | public async Task IngestToAdx2Async(string batchResponse, string tableName, string filePrefix) 142 | { 143 | string fileName = string.Format(@"{0}.json", filePrefix); 144 | 145 | await AppendToBlobAsync(batchResponse, filePrefix); 146 | 147 | log.LogInformation($"IngestionUri: {ingestionUri}"); 148 | var ingestConnectionStringBuilder = new KustoConnectionStringBuilder(ingestionUri, databaseName).WithAadUserManagedIdentity(msiClientId); 149 | 150 | // Client should be static 151 | // Create a disposable client that will execute the ingestion 152 | //TODO: Is above something that needs to be done? 153 | using (IKustoQueuedIngestClient client = KustoIngestFactory.CreateQueuedIngestClient(ingestConnectionStringBuilder)) 154 | { 155 | log.LogInformation($"IngestClient: {client}"); 156 | //Ingest from blobs according to the required properties 157 | var kustoIngestionProperties = new KustoQueuedIngestionProperties(databaseName: databaseName, tableName: $"{tableName}_Raw") 158 | { 159 | Format = DataSourceFormat.multijson, 160 | IngestionMapping = new IngestionMapping() 161 | { 162 | IngestionMappingReference = "RawMetricsMapping" 163 | }, 164 | FlushImmediately = false 165 | }; 166 | 167 | var sourceOptions = new StorageSourceOptions() 168 | { 169 | DeleteSourceOnSuccess = false 170 | }; 171 | 172 | await client.IngestFromStorageAsync($"https://{storageAccountName}.blob.core.windows.net/{containerName}/{fileName};managed_identity=system", ingestionProperties: kustoIngestionProperties, sourceOptions); 173 | log.LogInformation($"Ingested data from {fileName} to {tableName}_Raw with MSI Object Id {kustoMSIObjectId}"); 174 | } 175 | } 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /Utils/scripts/dashboard_templates/Keyvault-1679088939482.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 84, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "grafana-azure-data-explorer-datasource", 34 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 35 | }, 36 | "fieldConfig": { 37 | "defaults": { 38 | "color": { 39 | "mode": "palette-classic" 40 | }, 41 | "custom": { 42 | "align": "auto", 43 | "cellOptions": { 44 | "type": "color-text" 45 | }, 46 | "filterable": true, 47 | "inspect": true 48 | }, 49 | "decimals": 3, 50 | "mappings": [], 51 | "thresholds": { 52 | "mode": "absolute", 53 | "steps": [ 54 | { 55 | "color": "dark-red", 56 | "value": null 57 | }, 58 | { 59 | "color": "dark-green", 60 | "value": 80 61 | } 62 | ] 63 | } 64 | }, 65 | "overrides": [ 66 | { 67 | "matcher": { 68 | "id": "byName", 69 | "options": "id" 70 | }, 71 | "properties": [ 72 | { 73 | "id": "links", 74 | "value": [ 75 | { 76 | "targetBlank": true, 77 | "title": "", 78 | "url": "https://portal.azure.com/${__data.fields.tenantDomain}/resource${__value.raw}/overview" 79 | } 80 | ] 81 | } 82 | ] 83 | } 84 | ] 85 | }, 86 | "gridPos": { 87 | "h": 13, 88 | "w": 24, 89 | "x": 0, 90 | "y": 0 91 | }, 92 | "id": 2, 93 | "options": { 94 | "cellHeight": "sm", 95 | "footer": { 96 | "countRows": false, 97 | "fields": "", 98 | "reducer": [ 99 | "sum" 100 | ], 101 | "show": false 102 | }, 103 | "showHeader": true 104 | }, 105 | "pluginVersion": "9.5.13", 106 | "targets": [ 107 | { 108 | "database": "multi002-metricsdb", 109 | "datasource": { 110 | "type": "grafana-azure-data-explorer-datasource", 111 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 112 | }, 113 | "expression": { 114 | "from": { 115 | "property": { 116 | "name": "Vm_Availability", 117 | "type": "string" 118 | }, 119 | "type": "property" 120 | }, 121 | "groupBy": { 122 | "expressions": [], 123 | "type": "and" 124 | }, 125 | "reduce": { 126 | "expressions": [], 127 | "type": "and" 128 | }, 129 | "where": { 130 | "expressions": [], 131 | "type": "and" 132 | } 133 | }, 134 | "pluginVersion": "4.5.0", 135 | "query": "let v2 = Keyvault_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100 and subscriptionId in ($Subscriptions)\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v1 = Keyvault_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v4 = Keyvault_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and subscriptionId in ($Subscriptions) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v3 = Keyvault_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nunion kind=outer (v3 | where \"All\" in ($Subscriptions) and \"All\" in ($Region)),\n(v1 | where \"All\" in ($Subscriptions) and \"All\" !in ($Region)),\n(v4 | where \"All\" !in ($Subscriptions) and \"All\" in ($Region)),\n(v2 | where \"All\" !in ($Subscriptions) and \"All\" !in ($Region))\n", 136 | "querySource": "raw", 137 | "queryType": "KQL", 138 | "rawMode": true, 139 | "refId": "A", 140 | "resultFormat": "table" 141 | } 142 | ], 143 | "title": "# of Keyvaults with Availability < 100", 144 | "type": "table" 145 | } 146 | ], 147 | "refresh": "", 148 | "schemaVersion": 38, 149 | "style": "dark", 150 | "tags": [], 151 | "templating": { 152 | "list": [ 153 | { 154 | "current": { 155 | "selected": false, 156 | "text": "2023-01-12 00:00:00", 157 | "value": "2023-01-12 00:00:00" 158 | }, 159 | "hide": 2, 160 | "name": "selecteddate", 161 | "options": [ 162 | { 163 | "selected": true, 164 | "text": "2023-01-12 00:00:00", 165 | "value": "2023-01-12 00:00:00" 166 | } 167 | ], 168 | "query": "2023-01-12 00:00:00", 169 | "skipUrlSync": false, 170 | "type": "textbox" 171 | }, 172 | { 173 | "current": { 174 | "isNone": true, 175 | "selected": false, 176 | "text": "None", 177 | "value": "" 178 | }, 179 | "datasource": { 180 | "type": "grafana-azure-data-explorer-datasource", 181 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 182 | }, 183 | "definition": "", 184 | "hide": 2, 185 | "includeAll": false, 186 | "multi": true, 187 | "name": "Region", 188 | "options": [], 189 | "query": "", 190 | "refresh": 1, 191 | "regex": "", 192 | "skipUrlSync": false, 193 | "sort": 0, 194 | "type": "query" 195 | }, 196 | { 197 | "current": { 198 | "isNone": true, 199 | "selected": false, 200 | "text": "None", 201 | "value": "" 202 | }, 203 | "datasource": { 204 | "type": "grafana-azure-data-explorer-datasource", 205 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 206 | }, 207 | "definition": "", 208 | "hide": 2, 209 | "includeAll": false, 210 | "multi": true, 211 | "name": "Subscriptions", 212 | "options": [], 213 | "query": "", 214 | "refresh": 1, 215 | "regex": "", 216 | "skipUrlSync": false, 217 | "sort": 0, 218 | "type": "query" 219 | }, 220 | { 221 | "current": { 222 | "isNone": true, 223 | "selected": false, 224 | "text": "None", 225 | "value": "" 226 | }, 227 | "datasource": { 228 | "type": "grafana-azure-data-explorer-datasource", 229 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 230 | }, 231 | "definition": "", 232 | "hide": 2, 233 | "includeAll": false, 234 | "multi": true, 235 | "name": "Solution", 236 | "options": [], 237 | "query": "", 238 | "refresh": 1, 239 | "regex": "", 240 | "skipUrlSync": false, 241 | "sort": 0, 242 | "type": "query" 243 | } 244 | ] 245 | }, 246 | "time": { 247 | "from": "now-6h", 248 | "to": "now" 249 | }, 250 | "timepicker": {}, 251 | "timezone": "utc", 252 | "title": "Keyvault", 253 | "uid": "e3c65d8f-73f2-44db-a13e-2f37494d10a8", 254 | "version": 2, 255 | "weekStart": "" 256 | } -------------------------------------------------------------------------------- /Utils/scripts/dashboard_templates/Loadbalancer-1679088952762.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 86, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "grafana-azure-data-explorer-datasource", 34 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 35 | }, 36 | "fieldConfig": { 37 | "defaults": { 38 | "color": { 39 | "mode": "palette-classic" 40 | }, 41 | "custom": { 42 | "align": "auto", 43 | "cellOptions": { 44 | "type": "color-text" 45 | }, 46 | "filterable": true, 47 | "inspect": true 48 | }, 49 | "decimals": 3, 50 | "mappings": [], 51 | "thresholds": { 52 | "mode": "absolute", 53 | "steps": [ 54 | { 55 | "color": "dark-red", 56 | "value": null 57 | }, 58 | { 59 | "color": "dark-green", 60 | "value": 100 61 | } 62 | ] 63 | } 64 | }, 65 | "overrides": [ 66 | { 67 | "matcher": { 68 | "id": "byName", 69 | "options": "id" 70 | }, 71 | "properties": [ 72 | { 73 | "id": "links", 74 | "value": [ 75 | { 76 | "targetBlank": true, 77 | "title": "", 78 | "url": "https://portal.azure.com/${__data.fields.tenantDomain}/resource${__value.raw}/overview" 79 | } 80 | ] 81 | } 82 | ] 83 | } 84 | ] 85 | }, 86 | "gridPos": { 87 | "h": 12, 88 | "w": 24, 89 | "x": 0, 90 | "y": 0 91 | }, 92 | "id": 2, 93 | "options": { 94 | "cellHeight": "sm", 95 | "footer": { 96 | "countRows": false, 97 | "fields": "", 98 | "reducer": [ 99 | "sum" 100 | ], 101 | "show": false 102 | }, 103 | "showHeader": true 104 | }, 105 | "pluginVersion": "9.5.13", 106 | "targets": [ 107 | { 108 | "database": "multi002-metricsdb", 109 | "datasource": { 110 | "type": "grafana-azure-data-explorer-datasource", 111 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 112 | }, 113 | "expression": { 114 | "from": { 115 | "property": { 116 | "name": "Vm_Availability", 117 | "type": "string" 118 | }, 119 | "type": "property" 120 | }, 121 | "groupBy": { 122 | "expressions": [], 123 | "type": "and" 124 | }, 125 | "reduce": { 126 | "expressions": [], 127 | "type": "and" 128 | }, 129 | "where": { 130 | "expressions": [], 131 | "type": "and" 132 | } 133 | }, 134 | "pluginVersion": "4.5.0", 135 | "query": "let v2 = Loadbalancer_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100 and subscriptionId in ($Subscriptions)\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v1 = Loadbalancer_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v4 = Loadbalancer_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and subscriptionId in ($Subscriptions) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v3 = Loadbalancer_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nunion kind=outer (v3 | where \"All\" in ($Subscriptions) and \"All\" in ($Region)),\n(v1 | where \"All\" in ($Subscriptions) and \"All\" !in ($Region)),\n(v4 | where \"All\" !in ($Subscriptions) and \"All\" in ($Region)),\n(v2 | where \"All\" !in ($Subscriptions) and \"All\" !in ($Region))\n", 136 | "querySource": "raw", 137 | "queryType": "KQL", 138 | "rawMode": true, 139 | "refId": "A", 140 | "resultFormat": "table" 141 | } 142 | ], 143 | "title": "# of Loadbalancers with Availability < 100", 144 | "type": "table" 145 | } 146 | ], 147 | "refresh": "", 148 | "schemaVersion": 38, 149 | "style": "dark", 150 | "tags": [], 151 | "templating": { 152 | "list": [ 153 | { 154 | "current": { 155 | "selected": false, 156 | "text": "2023-01-12 00:00:00", 157 | "value": "2023-01-12 00:00:00" 158 | }, 159 | "hide": 2, 160 | "name": "selecteddate", 161 | "options": [ 162 | { 163 | "selected": true, 164 | "text": "2023-01-12 00:00:00", 165 | "value": "2023-01-12 00:00:00" 166 | } 167 | ], 168 | "query": "2023-01-12 00:00:00", 169 | "skipUrlSync": false, 170 | "type": "textbox" 171 | }, 172 | { 173 | "current": { 174 | "isNone": true, 175 | "selected": false, 176 | "text": "None", 177 | "value": "" 178 | }, 179 | "datasource": { 180 | "type": "grafana-azure-data-explorer-datasource", 181 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 182 | }, 183 | "definition": "", 184 | "hide": 2, 185 | "includeAll": false, 186 | "multi": true, 187 | "name": "Region", 188 | "options": [], 189 | "query": "", 190 | "refresh": 1, 191 | "regex": "", 192 | "skipUrlSync": false, 193 | "sort": 0, 194 | "type": "query" 195 | }, 196 | { 197 | "current": { 198 | "isNone": true, 199 | "selected": false, 200 | "text": "None", 201 | "value": "" 202 | }, 203 | "datasource": { 204 | "type": "grafana-azure-data-explorer-datasource", 205 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 206 | }, 207 | "definition": "", 208 | "hide": 2, 209 | "includeAll": false, 210 | "multi": true, 211 | "name": "Subscriptions", 212 | "options": [], 213 | "query": "", 214 | "refresh": 1, 215 | "regex": "", 216 | "skipUrlSync": false, 217 | "sort": 0, 218 | "type": "query" 219 | }, 220 | { 221 | "current": { 222 | "isNone": true, 223 | "selected": false, 224 | "text": "None", 225 | "value": "" 226 | }, 227 | "datasource": { 228 | "type": "grafana-azure-data-explorer-datasource", 229 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 230 | }, 231 | "definition": "", 232 | "hide": 2, 233 | "includeAll": false, 234 | "multi": true, 235 | "name": "Solution", 236 | "options": [], 237 | "query": "", 238 | "refresh": 1, 239 | "regex": "", 240 | "skipUrlSync": false, 241 | "sort": 0, 242 | "type": "query" 243 | } 244 | ] 245 | }, 246 | "time": { 247 | "from": "now-6h", 248 | "to": "now" 249 | }, 250 | "timepicker": {}, 251 | "timezone": "utc", 252 | "title": "Loadbalancer", 253 | "uid": "edbbddfb-a8d7-4b5d-bfa4-9703ac678f2b", 254 | "version": 2, 255 | "weekStart": "" 256 | } -------------------------------------------------------------------------------- /Utils/scripts/dashboard_templates/LogAnalytics-1688018903992.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 85, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "grafana-azure-data-explorer-datasource", 34 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 35 | }, 36 | "fieldConfig": { 37 | "defaults": { 38 | "color": { 39 | "mode": "palette-classic" 40 | }, 41 | "custom": { 42 | "align": "auto", 43 | "cellOptions": { 44 | "type": "color-text" 45 | }, 46 | "filterable": true, 47 | "inspect": true 48 | }, 49 | "decimals": 3, 50 | "mappings": [], 51 | "thresholds": { 52 | "mode": "absolute", 53 | "steps": [ 54 | { 55 | "color": "dark-red", 56 | "value": null 57 | }, 58 | { 59 | "color": "dark-green", 60 | "value": 100 61 | } 62 | ] 63 | } 64 | }, 65 | "overrides": [ 66 | { 67 | "matcher": { 68 | "id": "byName", 69 | "options": "id" 70 | }, 71 | "properties": [ 72 | { 73 | "id": "links", 74 | "value": [ 75 | { 76 | "targetBlank": true, 77 | "title": "", 78 | "url": "https://portal.azure.com/${__data.fields.tenantDomain}/resource${__value.raw}/overview" 79 | } 80 | ] 81 | } 82 | ] 83 | } 84 | ] 85 | }, 86 | "gridPos": { 87 | "h": 12, 88 | "w": 24, 89 | "x": 0, 90 | "y": 0 91 | }, 92 | "id": 2, 93 | "options": { 94 | "cellHeight": "sm", 95 | "footer": { 96 | "countRows": false, 97 | "fields": "", 98 | "reducer": [ 99 | "sum" 100 | ], 101 | "show": false 102 | }, 103 | "showHeader": true 104 | }, 105 | "pluginVersion": "9.5.13", 106 | "targets": [ 107 | { 108 | "database": "multi002-metricsdb", 109 | "datasource": { 110 | "type": "grafana-azure-data-explorer-datasource", 111 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 112 | }, 113 | "expression": { 114 | "from": { 115 | "property": { 116 | "name": "Vm_Availability", 117 | "type": "string" 118 | }, 119 | "type": "property" 120 | }, 121 | "groupBy": { 122 | "expressions": [], 123 | "type": "and" 124 | }, 125 | "reduce": { 126 | "expressions": [], 127 | "type": "and" 128 | }, 129 | "where": { 130 | "expressions": [], 131 | "type": "and" 132 | } 133 | }, 134 | "pluginVersion": "4.4.1", 135 | "query": "let v2 = LogAnalytics_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100 and subscriptionId in ($Subscriptions)\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v1 = LogAnalytics_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v4 = LogAnalytics_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and subscriptionId in ($Subscriptions) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v3 = LogAnalytics_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nunion kind=outer (v3 | where \"All\" in ($Subscriptions) and \"All\" in ($Region)),\n(v1 | where \"All\" in ($Subscriptions) and \"All\" !in ($Region)),\n(v4 | where \"All\" !in ($Subscriptions) and \"All\" in ($Region)),\n(v2 | where \"All\" !in ($Subscriptions) and \"All\" !in ($Region))\n", 136 | "querySource": "raw", 137 | "queryType": "KQL", 138 | "rawMode": true, 139 | "refId": "A", 140 | "resultFormat": "table" 141 | } 142 | ], 143 | "title": "# of LogAnalytics with Availability < 100", 144 | "type": "table" 145 | } 146 | ], 147 | "refresh": "", 148 | "revision": 1, 149 | "schemaVersion": 38, 150 | "style": "dark", 151 | "tags": [], 152 | "templating": { 153 | "list": [ 154 | { 155 | "current": { 156 | "selected": false, 157 | "text": "2023-01-12 00:00:00", 158 | "value": "2023-01-12 00:00:00" 159 | }, 160 | "hide": 2, 161 | "name": "selecteddate", 162 | "options": [ 163 | { 164 | "selected": true, 165 | "text": "2023-01-12 00:00:00", 166 | "value": "2023-01-12 00:00:00" 167 | } 168 | ], 169 | "query": "2023-01-12 00:00:00", 170 | "skipUrlSync": false, 171 | "type": "textbox" 172 | }, 173 | { 174 | "current": { 175 | "isNone": true, 176 | "selected": false, 177 | "text": "None", 178 | "value": "" 179 | }, 180 | "datasource": { 181 | "type": "grafana-azure-data-explorer-datasource", 182 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 183 | }, 184 | "definition": "", 185 | "hide": 2, 186 | "includeAll": false, 187 | "multi": true, 188 | "name": "Region", 189 | "options": [], 190 | "query": "", 191 | "refresh": 1, 192 | "regex": "", 193 | "skipUrlSync": false, 194 | "sort": 0, 195 | "type": "query" 196 | }, 197 | { 198 | "current": { 199 | "isNone": true, 200 | "selected": false, 201 | "text": "None", 202 | "value": "" 203 | }, 204 | "datasource": { 205 | "type": "grafana-azure-data-explorer-datasource", 206 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 207 | }, 208 | "definition": "", 209 | "hide": 2, 210 | "includeAll": false, 211 | "multi": true, 212 | "name": "Subscriptions", 213 | "options": [], 214 | "query": "", 215 | "refresh": 1, 216 | "regex": "", 217 | "skipUrlSync": false, 218 | "sort": 0, 219 | "type": "query" 220 | }, 221 | { 222 | "current": { 223 | "isNone": true, 224 | "selected": false, 225 | "text": "None", 226 | "value": "" 227 | }, 228 | "datasource": { 229 | "type": "grafana-azure-data-explorer-datasource", 230 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 231 | }, 232 | "definition": "", 233 | "hide": 2, 234 | "includeAll": false, 235 | "multi": true, 236 | "name": "Solution", 237 | "options": [], 238 | "query": "", 239 | "refresh": 1, 240 | "regex": "", 241 | "skipUrlSync": false, 242 | "sort": 0, 243 | "type": "query" 244 | } 245 | ] 246 | }, 247 | "time": { 248 | "from": "now-6h", 249 | "to": "now" 250 | }, 251 | "timepicker": {}, 252 | "timezone": "utc", 253 | "title": "LogAnalytics", 254 | "uid": "mpNZfMr4k", 255 | "version": 2, 256 | "weekStart": "" 257 | } -------------------------------------------------------------------------------- /Utils/scripts/dashboard_templates/Eventhubs-1687851669082.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 89, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "grafana-azure-data-explorer-datasource", 34 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 35 | }, 36 | "fieldConfig": { 37 | "defaults": { 38 | "color": { 39 | "mode": "palette-classic" 40 | }, 41 | "custom": { 42 | "align": "auto", 43 | "cellOptions": { 44 | "type": "color-text" 45 | }, 46 | "filterable": true, 47 | "inspect": true 48 | }, 49 | "decimals": 3, 50 | "mappings": [], 51 | "thresholds": { 52 | "mode": "absolute", 53 | "steps": [ 54 | { 55 | "color": "dark-red", 56 | "value": null 57 | }, 58 | { 59 | "color": "dark-green", 60 | "value": 100 61 | } 62 | ] 63 | } 64 | }, 65 | "overrides": [ 66 | { 67 | "matcher": { 68 | "id": "byName", 69 | "options": "id" 70 | }, 71 | "properties": [ 72 | { 73 | "id": "links", 74 | "value": [ 75 | { 76 | "targetBlank": true, 77 | "title": "", 78 | "url": "https://portal.azure.com/${__data.fields.tenantDomain}/resource${__value.raw}/overview" 79 | } 80 | ] 81 | } 82 | ] 83 | } 84 | ] 85 | }, 86 | "gridPos": { 87 | "h": 12, 88 | "w": 24, 89 | "x": 0, 90 | "y": 0 91 | }, 92 | "id": 2, 93 | "options": { 94 | "cellHeight": "sm", 95 | "footer": { 96 | "countRows": false, 97 | "fields": "", 98 | "reducer": [ 99 | "sum" 100 | ], 101 | "show": false 102 | }, 103 | "showHeader": true 104 | }, 105 | "pluginVersion": "9.5.13", 106 | "targets": [ 107 | { 108 | "database": "multi002-metricsdb", 109 | "datasource": { 110 | "type": "grafana-azure-data-explorer-datasource", 111 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 112 | }, 113 | "expression": { 114 | "from": { 115 | "property": { 116 | "name": "Vm_Availability", 117 | "type": "string" 118 | }, 119 | "type": "property" 120 | }, 121 | "groupBy": { 122 | "expressions": [], 123 | "type": "and" 124 | }, 125 | "reduce": { 126 | "expressions": [], 127 | "type": "and" 128 | }, 129 | "where": { 130 | "expressions": [], 131 | "type": "and" 132 | } 133 | }, 134 | "pluginVersion": "4.4.1", 135 | "query": "let v2 = Eventhubs_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project ['date'], id, location, name, incomingRequests, serverErrors,\n subscriptionId,tenantDomain, availability = (incomingRequests-serverErrors)*100/incomingRequests\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100 and subscriptionId in ($Subscriptions)\n| order by availability asc \n| order by ['date'] asc;\nlet v1 = Eventhubs_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project ['date'], id, location, name, incomingRequests, serverErrors,\n subscriptionId, tenantDomain, availability = (incomingRequests-serverErrors)*100/incomingRequests\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100\n| order by availability asc \n| order by ['date'] asc;\nlet v4 = Eventhubs_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project ['date'], id, location, name, incomingRequests, serverErrors,\n subscriptionId, tenantDomain, availability = (incomingRequests-serverErrors)*100/incomingRequests\n| where ['date'] == datetime($selecteddate) and subscriptionId in ($Subscriptions) and isnotnull(availability) and availability < 100 \n| order by availability asc \n| order by ['date'] asc;\nlet v3 = Eventhubs_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project ['date'], id, location, name, incomingRequests, serverErrors,\n subscriptionId, tenantDomain, availability = (incomingRequests-serverErrors)*100/incomingRequests\n| where ['date'] == datetime($selecteddate) and isnotnull(availability) and availability < 100 \n| order by availability asc \n| order by ['date'] asc;\nunion kind=outer (v3 | where \"All\" in ($Subscriptions) and \"All\" in ($Region)),\n(v1 | where \"All\" in ($Subscriptions) and \"All\" !in ($Region)),\n(v4 | where \"All\" !in ($Subscriptions) and \"All\" in ($Region)),\n(v2 | where \"All\" !in ($Subscriptions) and \"All\" !in ($Region))", 136 | "querySource": "raw", 137 | "queryType": "KQL", 138 | "rawMode": true, 139 | "refId": "A", 140 | "resultFormat": "table" 141 | } 142 | ], 143 | "title": "# of Eventhubs with Availability < 100", 144 | "type": "table" 145 | } 146 | ], 147 | "refresh": "", 148 | "revision": 1, 149 | "schemaVersion": 38, 150 | "style": "dark", 151 | "tags": [], 152 | "templating": { 153 | "list": [ 154 | { 155 | "current": { 156 | "selected": false, 157 | "text": "2023-01-12 00:00:00", 158 | "value": "2023-01-12 00:00:00" 159 | }, 160 | "hide": 2, 161 | "name": "selecteddate", 162 | "options": [ 163 | { 164 | "selected": true, 165 | "text": "2023-01-12 00:00:00", 166 | "value": "2023-01-12 00:00:00" 167 | } 168 | ], 169 | "query": "2023-01-12 00:00:00", 170 | "skipUrlSync": false, 171 | "type": "textbox" 172 | }, 173 | { 174 | "current": { 175 | "isNone": true, 176 | "selected": false, 177 | "text": "None", 178 | "value": "" 179 | }, 180 | "datasource": { 181 | "type": "grafana-azure-data-explorer-datasource", 182 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 183 | }, 184 | "definition": "", 185 | "hide": 2, 186 | "includeAll": false, 187 | "multi": true, 188 | "name": "Region", 189 | "options": [], 190 | "query": "", 191 | "refresh": 1, 192 | "regex": "", 193 | "skipUrlSync": false, 194 | "sort": 0, 195 | "type": "query" 196 | }, 197 | { 198 | "current": { 199 | "isNone": true, 200 | "selected": false, 201 | "text": "None", 202 | "value": "" 203 | }, 204 | "datasource": { 205 | "type": "grafana-azure-data-explorer-datasource", 206 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 207 | }, 208 | "definition": "", 209 | "hide": 2, 210 | "includeAll": false, 211 | "multi": true, 212 | "name": "Subscriptions", 213 | "options": [], 214 | "query": "", 215 | "refresh": 1, 216 | "regex": "", 217 | "skipUrlSync": false, 218 | "sort": 0, 219 | "type": "query" 220 | }, 221 | { 222 | "current": { 223 | "isNone": true, 224 | "selected": false, 225 | "text": "None", 226 | "value": "" 227 | }, 228 | "datasource": { 229 | "type": "grafana-azure-data-explorer-datasource", 230 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 231 | }, 232 | "definition": "", 233 | "hide": 2, 234 | "includeAll": false, 235 | "multi": true, 236 | "name": "Solution", 237 | "options": [], 238 | "query": "", 239 | "refresh": 1, 240 | "regex": "", 241 | "skipUrlSync": false, 242 | "sort": 0, 243 | "type": "query" 244 | } 245 | ] 246 | }, 247 | "time": { 248 | "from": "now-6h", 249 | "to": "now" 250 | }, 251 | "timepicker": {}, 252 | "timezone": "utc", 253 | "title": "Eventhubs", 254 | "uid": "K9cdIeu4z", 255 | "version": 2, 256 | "weekStart": "" 257 | } -------------------------------------------------------------------------------- /AdxIngestFunctionApp/AdxIngestFunction.cs: -------------------------------------------------------------------------------- 1 | using Azure.Core; 2 | using Azure.Identity; 3 | using Observability.Utils; 4 | using Observability.Utils.Data; 5 | using Microsoft.Azure.WebJobs; 6 | using Microsoft.Extensions.Configuration; 7 | using Microsoft.Extensions.Logging; 8 | using System; 9 | using System.Linq; 10 | using System.Net; 11 | using System.Net.Http; 12 | using System.Net.Http.Headers; 13 | using System.Text; 14 | using System.Threading.Tasks; 15 | using Microsoft.Azure.Management.ResourceManager.Fluent.Models; 16 | 17 | 18 | namespace Observability.AdxIngestFunctionApp 19 | { 20 | 21 | public class AdxIngestFunction 22 | { 23 | private static HttpClient _httpClient = new HttpClient(); 24 | private static KeyVaultManager keyVaultManager = null; 25 | private static readonly IConfiguration _config = new ConfigurationBuilder().AddEnvironmentVariables().Build(); 26 | 27 | static AdxIngestFunction() 28 | { 29 | string DefaultRequestHeaders = _config.GetValue("DefaultRequestHeaders"); 30 | _httpClient.DefaultRequestHeaders.UserAgent.TryParseAdd(DefaultRequestHeaders); 31 | } 32 | 33 | [FunctionName("AdxIngestFunction")] 34 | public static async Task Run([ServiceBusTrigger("%queueName%", Connection = "ServiceBusConnection", IsSessionsEnabled = false)] String myQueueItem, ILogger log) 35 | { 36 | ClientSecretCredential spCredential; 37 | 38 | AccessToken accessToken; 39 | 40 | //TODO: Add Debug Asserts for parameters etc. But also check in realease? 41 | //Debug.Assert(descriptor.Name != null); 42 | //Debug.Assert(store != null); 43 | 44 | log.LogInformation($"AdxIngestFunction processing message: {myQueueItem}"); 45 | 46 | var config = new ConfigurationBuilder().AddEnvironmentVariables().Build(); //TODO: Consider moving to static AdxIngestionFunction 47 | var message = System.Text.Json.JsonSerializer.Deserialize(myQueueItem); 48 | var resourceIds = message.Resources.Select(r => r.ID).ToList(); 49 | var jsonResouces = System.Text.Json.JsonSerializer.Serialize(new { resourceids = resourceIds }); 50 | 51 | var timeSpan = message.From.ToString("yyyy-MM-ddTHH:mm:ss.fffZ") + "/" + message.To.ToString("yyyy-MM-ddTHH:mm:ss.fffZ"); 52 | 53 | var batchUrl = $"https://{message.Location}.metrics.monitor.azure.com/subscriptions/{message.SubscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=Availability&aggregation=average&metricNamespace={message.Type}&autoadjusttimegrain=true&api-version=2023-03-01-preview"; 54 | 55 | if (message.Type == "microsoft.network/azurefirewalls") 56 | { 57 | batchUrl = $"https://{message.Location}.metrics.monitor.azure.com/subscriptions/{message.SubscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=FirewallHealth&aggregation=average&metricNamespace={message.Type}&autoadjusttimegrain=true&api-version=2023-03-01-preview"; 58 | } 59 | if (message.Type == "microsoft.network/loadbalancers") 60 | { 61 | batchUrl = $"https://{message.Location}.metrics.monitor.azure.com/subscriptions/{message.SubscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=VipAvailability&aggregation=average&metricNamespace={message.Type}&autoadjusttimegrain=true&api-version=2023-03-01-preview"; 62 | } 63 | if (message.Type == "microsoft.containerservice/managedclusters") 64 | { 65 | batchUrl = $"https://{message.Location}.metrics.monitor.azure.com/subscriptions/{message.SubscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=kube_node_status_condition&aggregation=average&metricNamespace={message.Type}&filter=status2 eq '*'&validatedimensions=false&autoadjusttimegrain=true&api-version=2023-03-01-preview"; 66 | } 67 | if (message.Type == "microsoft.documentdb/databaseaccounts") 68 | { 69 | batchUrl = $"https://{message.Location}.metrics.monitor.azure.com/subscriptions/{message.SubscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=ServiceAvailability&aggregation=average&metricNamespace={message.Type}&autoadjusttimegrain=true&api-version=2023-03-01-preview"; 70 | } 71 | if (message.Type == "microsoft.cognitiveservices/accounts") 72 | { 73 | batchUrl = $"https://{message.Location}.metrics.monitor.azure.com/subscriptions/{message.SubscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=SuccessRate&aggregation=average&metricNamespace={message.Type}&autoadjusttimegrain=true&api-version=2023-03-01-preview"; 74 | } 75 | if (message.Type == "microsoft.eventhub/namespaces") 76 | { 77 | batchUrl = $"https://{message.Location}.metrics.monitor.azure.com/subscriptions/{message.SubscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=IncomingRequests,ServerErrors&aggregation=average&metricNamespace={message.Type}&autoadjusttimegrain=true&api-version=2023-03-01-preview"; 78 | } 79 | if (message.Type == "microsoft.containerregistry/registries") 80 | { 81 | batchUrl = $"https://{message.Location}.metrics.monitor.azure.com/subscriptions/{message.SubscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=SuccessfulPullCount,TotalPullCount,SuccessfulPushCount,TotalPushCount&aggregation=average&metricNamespace={message.Type}&autoadjusttimegrain=true&api-version=2023-03-01-preview"; 82 | } 83 | if (message.Type == "microsoft.operationalinsights/workspaces") 84 | { 85 | batchUrl = $"https://{message.Location}.metrics.monitor.azure.com/subscriptions/{message.SubscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=AvailabilityRate_Query&aggregation=average&metricNamespace={message.Type}&autoadjusttimegrain=true&api-version=2023-03-01-preview"; 86 | } 87 | log.LogInformation($"Batch url: {batchUrl}"); 88 | 89 | string currTime = timeSpan.Split('/')[0]; 90 | string currResource = message.Type.Substring(message.Type.IndexOf('/') + 1); 91 | string filePrefix = $"{message.Location}_{message.SubscriptionID}_{currTime}_{currResource}"; 92 | log.LogInformation($"Storage Blob File Prefix: {filePrefix}"); 93 | 94 | using HttpRequestMessage httpRequest = new HttpRequestMessage(HttpMethod.Post, batchUrl); 95 | 96 | string msftTenantId = config.GetValue("msftTenantId"); 97 | 98 | string tenantId = message.TenantId; 99 | 100 | 101 | 102 | // If Tenant Id from the Message Queue is not null 103 | 104 | if(tenantId != null && tenantId != msftTenantId) 105 | { 106 | log.LogInformation($"Tenant Id: {tenantId}"); 107 | if(keyVaultManager == null) 108 | { 109 | keyVaultManager = new KeyVaultManager(config, log); 110 | } 111 | Tenant tenant = keyVaultManager.GetServicePrincipalCredential(tenantId); 112 | string clientId = tenant.ClientId; 113 | string clientSecret = tenant.ClientSecret; 114 | log.LogInformation(clientId); 115 | 116 | spCredential = new ClientSecretCredential(tenantId, clientId, clientSecret); 117 | log.LogInformation("Done ClientSecretCredential"); 118 | 119 | accessToken = spCredential.GetToken(new TokenRequestContext(new[] { "https://metrics.monitor.azure.com/.default" })); 120 | log.LogInformation("Got accessToken"); 121 | 122 | } 123 | else 124 | { 125 | log.LogInformation($"MSFT Tenant Id: {tenantId}"); 126 | string userAssignedClientId = config.GetValue("msiclientId"); 127 | var credential = new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = userAssignedClientId }); 128 | accessToken = credential.GetToken(new TokenRequestContext(new[] { "https://metrics.monitor.azure.com/" })); 129 | } 130 | 131 | 132 | httpRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken.Token); 133 | 134 | httpRequest.Content = new StringContent(jsonResouces, Encoding.UTF8, "application/json"); 135 | 136 | using var response = await _httpClient.SendAsync(httpRequest); 137 | 138 | //TODO: Logging and not throwing. Confirm that this is correct approach 139 | if (response is { StatusCode: >= HttpStatusCode.BadRequest }) 140 | { 141 | log.LogInformation("Something went wrong with the Monitor API call"); 142 | log.LogInformation($"Response status code: {response.StatusCode}"); 143 | } 144 | log.LogInformation("Sucess response from Monitor API call"); 145 | 146 | var responseContent = await response.Content.ReadAsStringAsync(); //TODO: Should handle as stream and not bring into memory as a string. // see later converting string back to a stream in IngestToAdx2Async, AppendToBlobAsync 147 | 148 | var adx = new AdxClientHelper(config, log); 149 | await adx.IngestToAdx2Async(responseContent, message.ResultTable, filePrefix); 150 | } 151 | } 152 | } -------------------------------------------------------------------------------- /Utils/scripts/dashboard_templates/Firewalls-1689786810784.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 88, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "grafana-azure-data-explorer-datasource", 34 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 35 | }, 36 | "fieldConfig": { 37 | "defaults": { 38 | "color": { 39 | "mode": "palette-classic" 40 | }, 41 | "custom": { 42 | "align": "auto", 43 | "cellOptions": { 44 | "type": "color-text" 45 | }, 46 | "filterable": true, 47 | "inspect": true 48 | }, 49 | "decimals": 3, 50 | "mappings": [], 51 | "thresholds": { 52 | "mode": "absolute", 53 | "steps": [ 54 | { 55 | "color": "dark-red", 56 | "value": null 57 | }, 58 | { 59 | "color": "dark-green", 60 | "value": 100 61 | } 62 | ] 63 | } 64 | }, 65 | "overrides": [ 66 | { 67 | "matcher": { 68 | "id": "byName", 69 | "options": "id" 70 | }, 71 | "properties": [ 72 | { 73 | "id": "custom.width" 74 | }, 75 | { 76 | "id": "links", 77 | "value": [ 78 | { 79 | "targetBlank": true, 80 | "title": "", 81 | "url": "https://portal.azure.com/${__data.fields.tenantDomain}/resource${__value.raw}/overview" 82 | } 83 | ] 84 | } 85 | ] 86 | }, 87 | { 88 | "matcher": { 89 | "id": "byName", 90 | "options": "location" 91 | }, 92 | "properties": [ 93 | { 94 | "id": "custom.width", 95 | "value": 189 96 | } 97 | ] 98 | } 99 | ] 100 | }, 101 | "gridPos": { 102 | "h": 10, 103 | "w": 24, 104 | "x": 0, 105 | "y": 0 106 | }, 107 | "id": 2, 108 | "options": { 109 | "cellHeight": "sm", 110 | "footer": { 111 | "countRows": false, 112 | "fields": "", 113 | "reducer": [ 114 | "sum" 115 | ], 116 | "show": false 117 | }, 118 | "showHeader": true, 119 | "sortBy": [] 120 | }, 121 | "pluginVersion": "9.5.13", 122 | "targets": [ 123 | { 124 | "database": "multi002-metricsdb", 125 | "datasource": { 126 | "type": "grafana-azure-data-explorer-datasource", 127 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 128 | }, 129 | "expression": { 130 | "from": { 131 | "property": { 132 | "name": "Vm_Availability", 133 | "type": "string" 134 | }, 135 | "type": "property" 136 | }, 137 | "groupBy": { 138 | "expressions": [], 139 | "type": "and" 140 | }, 141 | "reduce": { 142 | "expressions": [], 143 | "type": "and" 144 | }, 145 | "where": { 146 | "expressions": [], 147 | "type": "and" 148 | } 149 | }, 150 | "pluginVersion": "4.5.0", 151 | "query": "let v2 = Firewall_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100 and subscriptionId in ($Subscriptions)\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v1 = Firewall_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v4 = Firewall_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and subscriptionId in ($Subscriptions) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v3 = Firewall_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nunion kind=outer (v3 | where \"All\" in ($Subscriptions) and \"All\" in ($Region)),\n(v1 | where \"All\" in ($Subscriptions) and \"All\" !in ($Region)),\n(v4 | where \"All\" !in ($Subscriptions) and \"All\" in ($Region)),\n(v2 | where \"All\" !in ($Subscriptions) and \"All\" !in ($Region))\n", 152 | "querySource": "raw", 153 | "queryType": "KQL", 154 | "rawMode": true, 155 | "refId": "A", 156 | "resultFormat": "table" 157 | } 158 | ], 159 | "title": "# of Firewall with Availability < 100", 160 | "type": "table" 161 | } 162 | ], 163 | "refresh": "", 164 | "revision": 1, 165 | "schemaVersion": 38, 166 | "style": "dark", 167 | "tags": [], 168 | "templating": { 169 | "list": [ 170 | { 171 | "current": { 172 | "selected": false, 173 | "text": "2023-01-12 00:00:00", 174 | "value": "2023-01-12 00:00:00" 175 | }, 176 | "hide": 2, 177 | "name": "selecteddate", 178 | "options": [ 179 | { 180 | "selected": true, 181 | "text": "2023-01-12 00:00:00", 182 | "value": "2023-01-12 00:00:00" 183 | } 184 | ], 185 | "query": "2023-01-12 00:00:00", 186 | "skipUrlSync": false, 187 | "type": "textbox" 188 | }, 189 | { 190 | "current": { 191 | "isNone": true, 192 | "selected": false, 193 | "text": "None", 194 | "value": "" 195 | }, 196 | "datasource": { 197 | "type": "grafana-azure-data-explorer-datasource", 198 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 199 | }, 200 | "definition": "", 201 | "hide": 2, 202 | "includeAll": false, 203 | "multi": true, 204 | "name": "Region", 205 | "options": [], 206 | "query": "", 207 | "refresh": 1, 208 | "regex": "", 209 | "skipUrlSync": false, 210 | "sort": 0, 211 | "type": "query" 212 | }, 213 | { 214 | "current": { 215 | "isNone": true, 216 | "selected": false, 217 | "text": "None", 218 | "value": "" 219 | }, 220 | "datasource": { 221 | "type": "grafana-azure-data-explorer-datasource", 222 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 223 | }, 224 | "definition": "", 225 | "hide": 2, 226 | "includeAll": false, 227 | "multi": true, 228 | "name": "Subscriptions", 229 | "options": [], 230 | "query": "", 231 | "refresh": 1, 232 | "regex": "", 233 | "skipUrlSync": false, 234 | "sort": 0, 235 | "type": "query" 236 | }, 237 | { 238 | "current": { 239 | "isNone": true, 240 | "selected": false, 241 | "text": "None", 242 | "value": "" 243 | }, 244 | "datasource": { 245 | "type": "grafana-azure-data-explorer-datasource", 246 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 247 | }, 248 | "definition": "", 249 | "hide": 2, 250 | "includeAll": false, 251 | "multi": true, 252 | "name": "Solution", 253 | "options": [], 254 | "query": "", 255 | "refresh": 1, 256 | "regex": "", 257 | "skipUrlSync": false, 258 | "sort": 0, 259 | "type": "query" 260 | } 261 | ] 262 | }, 263 | "time": { 264 | "from": "now-6h", 265 | "to": "now" 266 | }, 267 | "timepicker": {}, 268 | "timezone": "utc", 269 | "title": "Firewalls", 270 | "uid": "8HDuNUC4z", 271 | "version": 2, 272 | "weekStart": "" 273 | } -------------------------------------------------------------------------------- /Utils/scripts/dashboard_templates/Storage-1679088963314.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 92, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "grafana-azure-data-explorer-datasource", 34 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 35 | }, 36 | "description": "", 37 | "fieldConfig": { 38 | "defaults": { 39 | "color": { 40 | "mode": "palette-classic" 41 | }, 42 | "custom": { 43 | "align": "auto", 44 | "cellOptions": { 45 | "type": "color-text" 46 | }, 47 | "filterable": true, 48 | "inspect": true 49 | }, 50 | "decimals": 3, 51 | "mappings": [], 52 | "thresholds": { 53 | "mode": "absolute", 54 | "steps": [ 55 | { 56 | "color": "dark-red", 57 | "value": null 58 | }, 59 | { 60 | "color": "dark-green", 61 | "value": 100 62 | } 63 | ] 64 | } 65 | }, 66 | "overrides": [ 67 | { 68 | "matcher": { 69 | "id": "byName", 70 | "options": "id" 71 | }, 72 | "properties": [ 73 | { 74 | "id": "custom.width" 75 | }, 76 | { 77 | "id": "links", 78 | "value": [ 79 | { 80 | "targetBlank": true, 81 | "title": "", 82 | "url": "https://portal.azure.com/${__data.fields.tenantDomain}/resource${__value.raw}/overview" 83 | } 84 | ] 85 | } 86 | ] 87 | }, 88 | { 89 | "matcher": { 90 | "id": "byName", 91 | "options": "location" 92 | }, 93 | "properties": [ 94 | { 95 | "id": "custom.width", 96 | "value": 189 97 | } 98 | ] 99 | } 100 | ] 101 | }, 102 | "gridPos": { 103 | "h": 10, 104 | "w": 24, 105 | "x": 0, 106 | "y": 0 107 | }, 108 | "id": 2, 109 | "options": { 110 | "cellHeight": "sm", 111 | "footer": { 112 | "countRows": false, 113 | "fields": "", 114 | "reducer": [ 115 | "sum" 116 | ], 117 | "show": false 118 | }, 119 | "showHeader": true, 120 | "sortBy": [] 121 | }, 122 | "pluginVersion": "9.5.13", 123 | "targets": [ 124 | { 125 | "database": "multi002-metricsdb", 126 | "datasource": { 127 | "type": "grafana-azure-data-explorer-datasource", 128 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 129 | }, 130 | "expression": { 131 | "from": { 132 | "property": { 133 | "name": "Vm_Availability", 134 | "type": "string" 135 | }, 136 | "type": "property" 137 | }, 138 | "groupBy": { 139 | "expressions": [], 140 | "type": "and" 141 | }, 142 | "reduce": { 143 | "expressions": [], 144 | "type": "and" 145 | }, 146 | "where": { 147 | "expressions": [], 148 | "type": "and" 149 | } 150 | }, 151 | "pluginVersion": "4.5.0", 152 | "query": "let v2 = Storage_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100 and subscriptionId in ($Subscriptions)\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v1 = Storage_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v4 = Storage_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and subscriptionId in ($Subscriptions) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v3 = Storage_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nunion kind=outer (v3 | where \"All\" in ($Subscriptions) and \"All\" in ($Region)),\n(v1 | where \"All\" in ($Subscriptions) and \"All\" !in ($Region)),\n(v4 | where \"All\" !in ($Subscriptions) and \"All\" in ($Region)),\n(v2 | where \"All\" !in ($Subscriptions) and \"All\" !in ($Region))\n", 153 | "querySource": "raw", 154 | "queryType": "KQL", 155 | "rawMode": true, 156 | "refId": "A", 157 | "resultFormat": "table" 158 | } 159 | ], 160 | "title": "# of Storage with Availability < 100", 161 | "type": "table" 162 | } 163 | ], 164 | "refresh": "", 165 | "schemaVersion": 38, 166 | "style": "dark", 167 | "tags": [], 168 | "templating": { 169 | "list": [ 170 | { 171 | "current": { 172 | "selected": false, 173 | "text": "2023-01-12 00:00:00", 174 | "value": "2023-01-12 00:00:00" 175 | }, 176 | "hide": 2, 177 | "name": "selecteddate", 178 | "options": [ 179 | { 180 | "selected": true, 181 | "text": "2023-01-12 00:00:00", 182 | "value": "2023-01-12 00:00:00" 183 | } 184 | ], 185 | "query": "2023-01-12 00:00:00", 186 | "skipUrlSync": false, 187 | "type": "textbox" 188 | }, 189 | { 190 | "current": { 191 | "isNone": true, 192 | "selected": false, 193 | "text": "None", 194 | "value": "" 195 | }, 196 | "datasource": { 197 | "type": "grafana-azure-data-explorer-datasource", 198 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 199 | }, 200 | "definition": "", 201 | "hide": 2, 202 | "includeAll": false, 203 | "multi": true, 204 | "name": "Region", 205 | "options": [], 206 | "query": "", 207 | "refresh": 1, 208 | "regex": "", 209 | "skipUrlSync": false, 210 | "sort": 0, 211 | "type": "query" 212 | }, 213 | { 214 | "current": { 215 | "isNone": true, 216 | "selected": false, 217 | "text": "None", 218 | "value": "" 219 | }, 220 | "datasource": { 221 | "type": "grafana-azure-data-explorer-datasource", 222 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 223 | }, 224 | "definition": "", 225 | "hide": 2, 226 | "includeAll": false, 227 | "multi": true, 228 | "name": "Subscriptions", 229 | "options": [], 230 | "query": "", 231 | "refresh": 1, 232 | "regex": "", 233 | "skipUrlSync": false, 234 | "sort": 0, 235 | "type": "query" 236 | }, 237 | { 238 | "current": { 239 | "isNone": true, 240 | "selected": false, 241 | "text": "None", 242 | "value": "" 243 | }, 244 | "datasource": { 245 | "type": "grafana-azure-data-explorer-datasource", 246 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 247 | }, 248 | "definition": "", 249 | "hide": 2, 250 | "includeAll": false, 251 | "multi": true, 252 | "name": "Solution", 253 | "options": [], 254 | "query": "", 255 | "refresh": 1, 256 | "regex": "", 257 | "skipUrlSync": false, 258 | "sort": 0, 259 | "type": "query" 260 | } 261 | ] 262 | }, 263 | "time": { 264 | "from": "now-6h", 265 | "to": "now" 266 | }, 267 | "timepicker": {}, 268 | "timezone": "utc", 269 | "title": "Storage", 270 | "uid": "e90a362b-0e3e-4da7-a725-0f35415d18bc", 271 | "version": 2, 272 | "weekStart": "" 273 | } -------------------------------------------------------------------------------- /Utils/scripts/dashboard_templates/CosmosDB-1679088907885.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 90, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "grafana-azure-data-explorer-datasource", 34 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 35 | }, 36 | "fieldConfig": { 37 | "defaults": { 38 | "color": { 39 | "mode": "palette-classic" 40 | }, 41 | "custom": { 42 | "align": "auto", 43 | "cellOptions": { 44 | "type": "color-text" 45 | }, 46 | "filterable": true, 47 | "inspect": true 48 | }, 49 | "decimals": 3, 50 | "mappings": [], 51 | "thresholds": { 52 | "mode": "percentage", 53 | "steps": [ 54 | { 55 | "color": "semi-dark-red", 56 | "value": null 57 | }, 58 | { 59 | "color": "dark-green", 60 | "value": 100 61 | } 62 | ] 63 | } 64 | }, 65 | "overrides": [ 66 | { 67 | "matcher": { 68 | "id": "byName", 69 | "options": "availability" 70 | }, 71 | "properties": [ 72 | { 73 | "id": "custom.cellOptions", 74 | "value": { 75 | "type": "color-text" 76 | } 77 | } 78 | ] 79 | }, 80 | { 81 | "matcher": { 82 | "id": "byName", 83 | "options": "id" 84 | }, 85 | "properties": [ 86 | { 87 | "id": "links", 88 | "value": [ 89 | { 90 | "targetBlank": true, 91 | "title": "", 92 | "url": "https://portal.azure.com/${__data.fields.tenantDomain}/resource${__value.raw}/overview" 93 | } 94 | ] 95 | } 96 | ] 97 | } 98 | ] 99 | }, 100 | "gridPos": { 101 | "h": 12, 102 | "w": 24, 103 | "x": 0, 104 | "y": 0 105 | }, 106 | "id": 2, 107 | "options": { 108 | "cellHeight": "sm", 109 | "footer": { 110 | "countRows": false, 111 | "fields": "", 112 | "reducer": [ 113 | "sum" 114 | ], 115 | "show": false 116 | }, 117 | "showHeader": true 118 | }, 119 | "pluginVersion": "9.5.13", 120 | "targets": [ 121 | { 122 | "database": "multi002-metricsdb", 123 | "datasource": { 124 | "type": "grafana-azure-data-explorer-datasource", 125 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 126 | }, 127 | "expression": { 128 | "from": { 129 | "property": { 130 | "name": "vm_availability", 131 | "type": "string" 132 | }, 133 | "type": "property" 134 | }, 135 | "groupBy": { 136 | "expressions": [], 137 | "type": "and" 138 | }, 139 | "reduce": { 140 | "expressions": [], 141 | "type": "and" 142 | }, 143 | "where": { 144 | "expressions": [], 145 | "type": "and" 146 | } 147 | }, 148 | "pluginVersion": "4.5.0", 149 | "query": "let v2 = Cosmosdb_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100 and subscriptionId in ($Subscriptions)\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v1 = Cosmosdb_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and location in ($Region) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v4 = Cosmosdb_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and subscriptionId in ($Subscriptions) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v3 = Cosmosdb_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| where ['date'] == datetime($selecteddate) and isnotnull(availability) and availability < 100\n| project ['date'], subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nunion kind=outer (v3 | where \"All\" in ($Subscriptions) and \"All\" in ($Region)),\n(v1 | where \"All\" in ($Subscriptions) and \"All\" !in ($Region)),\n(v4 | where \"All\" !in ($Subscriptions) and \"All\" in ($Region)),\n(v2 | where \"All\" !in ($Subscriptions) and \"All\" !in ($Region))//(v1 | where \"All\" in ($Subscriptions)),(v2 | where \"All\" !in ($Subscriptions));\n", 150 | "querySource": "raw", 151 | "queryType": "KQL", 152 | "rawMode": true, 153 | "refId": "A", 154 | "resultFormat": "table" 155 | } 156 | ], 157 | "title": "# of CosmosDB with Availability < 100", 158 | "type": "table" 159 | } 160 | ], 161 | "refresh": "", 162 | "schemaVersion": 38, 163 | "style": "dark", 164 | "tags": [], 165 | "templating": { 166 | "list": [ 167 | { 168 | "current": { 169 | "selected": false, 170 | "text": "2023-01-12 00:00:00", 171 | "value": "2023-01-12 00:00:00" 172 | }, 173 | "hide": 2, 174 | "name": "selecteddate", 175 | "options": [ 176 | { 177 | "selected": true, 178 | "text": "2023-01-12 00:00:00", 179 | "value": "2023-01-12 00:00:00" 180 | } 181 | ], 182 | "query": "2023-01-12 00:00:00", 183 | "skipUrlSync": false, 184 | "type": "textbox" 185 | }, 186 | { 187 | "current": { 188 | "isNone": true, 189 | "selected": false, 190 | "text": "None", 191 | "value": "" 192 | }, 193 | "datasource": { 194 | "type": "grafana-azure-data-explorer-datasource", 195 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 196 | }, 197 | "definition": "", 198 | "hide": 2, 199 | "includeAll": false, 200 | "multi": true, 201 | "name": "Region", 202 | "options": [], 203 | "query": "", 204 | "refresh": 1, 205 | "regex": "", 206 | "skipUrlSync": false, 207 | "sort": 0, 208 | "type": "query" 209 | }, 210 | { 211 | "current": { 212 | "isNone": true, 213 | "selected": false, 214 | "text": "None", 215 | "value": "" 216 | }, 217 | "datasource": { 218 | "type": "grafana-azure-data-explorer-datasource", 219 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 220 | }, 221 | "definition": "", 222 | "hide": 2, 223 | "includeAll": false, 224 | "multi": true, 225 | "name": "Subscriptions", 226 | "options": [], 227 | "query": "", 228 | "refresh": 1, 229 | "regex": "", 230 | "skipUrlSync": false, 231 | "sort": 0, 232 | "type": "query" 233 | }, 234 | { 235 | "current": { 236 | "isNone": true, 237 | "selected": false, 238 | "text": "None", 239 | "value": "" 240 | }, 241 | "datasource": { 242 | "type": "grafana-azure-data-explorer-datasource", 243 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 244 | }, 245 | "definition": "", 246 | "hide": 2, 247 | "includeAll": false, 248 | "multi": true, 249 | "name": "Solution", 250 | "options": [], 251 | "query": "", 252 | "refresh": 1, 253 | "regex": "", 254 | "skipUrlSync": false, 255 | "sort": 0, 256 | "type": "query" 257 | } 258 | ] 259 | }, 260 | "time": { 261 | "from": "now-6h", 262 | "to": "now" 263 | }, 264 | "timepicker": {}, 265 | "timezone": "utc", 266 | "title": "CosmosDB", 267 | "uid": "a05004e1-8d3f-4e62-90e2-293853b8e41e", 268 | "version": 2, 269 | "weekStart": "" 270 | } -------------------------------------------------------------------------------- /Utils/scripts/dashboard_templates/AksServerNode-1679088882867.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 87, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "grafana-azure-data-explorer-datasource", 34 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 35 | }, 36 | "fieldConfig": { 37 | "defaults": { 38 | "color": { 39 | "mode": "palette-classic" 40 | }, 41 | "custom": { 42 | "align": "auto", 43 | "cellOptions": { 44 | "type": "color-text" 45 | }, 46 | "filterable": true, 47 | "inspect": true 48 | }, 49 | "decimals": 3, 50 | "mappings": [], 51 | "thresholds": { 52 | "mode": "absolute", 53 | "steps": [ 54 | { 55 | "color": "dark-red", 56 | "value": null 57 | }, 58 | { 59 | "color": "dark-green", 60 | "value": 100 61 | } 62 | ] 63 | } 64 | }, 65 | "overrides": [ 66 | { 67 | "matcher": { 68 | "id": "byName", 69 | "options": "id" 70 | }, 71 | "properties": [ 72 | { 73 | "id": "links", 74 | "value": [ 75 | { 76 | "targetBlank": true, 77 | "title": "", 78 | "url": "https://portal.azure.com/${__data.fields.tenantDomain}/resource${__value.raw}/overview" 79 | } 80 | ] 81 | } 82 | ] 83 | } 84 | ] 85 | }, 86 | "gridPos": { 87 | "h": 10, 88 | "w": 24, 89 | "x": 0, 90 | "y": 0 91 | }, 92 | "id": 2, 93 | "options": { 94 | "cellHeight": "sm", 95 | "footer": { 96 | "countRows": false, 97 | "fields": "", 98 | "reducer": [ 99 | "sum" 100 | ], 101 | "show": false 102 | }, 103 | "showHeader": true 104 | }, 105 | "pluginVersion": "9.5.13", 106 | "targets": [ 107 | { 108 | "database": "multi002-metricsdb", 109 | "datasource": { 110 | "type": "grafana-azure-data-explorer-datasource", 111 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 112 | }, 113 | "expression": { 114 | "from": { 115 | "property": { 116 | "name": "Vm_Availability", 117 | "type": "string" 118 | }, 119 | "type": "property" 120 | }, 121 | "groupBy": { 122 | "expressions": [], 123 | "type": "and" 124 | }, 125 | "reduce": { 126 | "expressions": [], 127 | "type": "and" 128 | }, 129 | "where": { 130 | "expressions": [], 131 | "type": "and" 132 | } 133 | }, 134 | "pluginVersion": "4.5.0", 135 | "query": "let v2=Aksservernode_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project component, createdAt, ['date'], id, location, name, nodeNotReady,\n nodeReady, nodeUnknown,solution, subscriptionId, tenantDomain, availability = ((nodeReady )/(nodeNotReady +nodeReady))*100\n| where $__timeFilter(['date'])\n and location in ($Region) and subscriptionId in ($Subscriptions)\n and solution in ($Solution) and availability < 100\n| project ['date'] , subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v1=Aksservernode_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project component, createdAt, ['date'], id, location, name, nodeNotReady,\n nodeReady, nodeUnknown,solution, subscriptionId, tenantDomain, availability = ((nodeReady )/(nodeNotReady +nodeReady))*100\n| where $__timeFilter(['date'])\nand location in ($Region)\nand solution in ($Solution) and availability < 100\n| project ['date'] , subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v4=Aksservernode_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project component, createdAt, ['date'], id, location, name, nodeNotReady,\n nodeReady, nodeUnknown,solution, subscriptionId, tenantDomain, availability = ((nodeReady )/(nodeNotReady +nodeReady))*100\n| where $__timeFilter(['date'])\n and subscriptionId in ($Subscriptions)\n and solution in ($Solution) and availability < 100\n| project ['date'] , subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v3=Aksservernode_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project component, createdAt, ['date'], id, location, name, nodeNotReady,\n nodeReady, nodeUnknown,solution, subscriptionId, tenantDomain, availability = ((nodeReady )/(nodeNotReady +nodeReady))*100\n| where $__timeFilter(['date'])\n and solution in ($Solution) and availability < 100\n| project ['date'] , subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nunion kind=outer (v3 | where \"All\" in ($Subscriptions) and \"All\" in ($Region)),\n(v1 | where \"All\" in ($Subscriptions) and \"All\" !in ($Region)),\n(v4 | where \"All\" !in ($Subscriptions) and \"All\" in ($Region)),\n(v2 | where \"All\" !in ($Subscriptions) and \"All\" !in ($Region))", 136 | "querySource": "raw", 137 | "queryType": "KQL", 138 | "rawMode": true, 139 | "refId": "A", 140 | "resultFormat": "table" 141 | } 142 | ], 143 | "title": "# of AksServerNode with Availability < 100", 144 | "type": "table" 145 | } 146 | ], 147 | "refresh": "", 148 | "schemaVersion": 38, 149 | "style": "dark", 150 | "tags": [], 151 | "templating": { 152 | "list": [ 153 | { 154 | "current": { 155 | "selected": false, 156 | "text": "2023-09-03 13:15:00", 157 | "value": "2023-09-03 13:15:00" 158 | }, 159 | "hide": 2, 160 | "name": "selecteddate", 161 | "options": [ 162 | { 163 | "selected": true, 164 | "text": "2023-09-03 13:15:00", 165 | "value": "2023-09-03 13:15:00" 166 | } 167 | ], 168 | "query": "2023-09-03 13:15:00", 169 | "skipUrlSync": false, 170 | "type": "textbox" 171 | }, 172 | { 173 | "current": { 174 | "isNone": true, 175 | "selected": false, 176 | "text": "None", 177 | "value": "" 178 | }, 179 | "datasource": { 180 | "type": "grafana-azure-data-explorer-datasource", 181 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 182 | }, 183 | "definition": "", 184 | "hide": 2, 185 | "includeAll": false, 186 | "multi": true, 187 | "name": "Subscriptions", 188 | "options": [], 189 | "query": "", 190 | "refresh": 1, 191 | "regex": "", 192 | "skipUrlSync": false, 193 | "sort": 0, 194 | "type": "query" 195 | }, 196 | { 197 | "current": { 198 | "isNone": true, 199 | "selected": false, 200 | "text": "None", 201 | "value": "" 202 | }, 203 | "datasource": { 204 | "type": "grafana-azure-data-explorer-datasource", 205 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 206 | }, 207 | "definition": "", 208 | "hide": 2, 209 | "includeAll": false, 210 | "multi": true, 211 | "name": "Region", 212 | "options": [], 213 | "query": "", 214 | "refresh": 1, 215 | "regex": "", 216 | "skipUrlSync": false, 217 | "sort": 0, 218 | "type": "query" 219 | }, 220 | { 221 | "current": { 222 | "isNone": true, 223 | "selected": false, 224 | "text": "None", 225 | "value": "" 226 | }, 227 | "datasource": { 228 | "type": "grafana-azure-data-explorer-datasource", 229 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 230 | }, 231 | "definition": "", 232 | "hide": 2, 233 | "includeAll": false, 234 | "multi": true, 235 | "name": "Solution", 236 | "options": [], 237 | "query": "", 238 | "refresh": 1, 239 | "regex": "", 240 | "skipUrlSync": false, 241 | "sort": 0, 242 | "type": "query" 243 | } 244 | ] 245 | }, 246 | "time": { 247 | "from": "now-6h", 248 | "to": "now" 249 | }, 250 | "timepicker": {}, 251 | "timezone": "utc", 252 | "title": "AksServerNode", 253 | "uid": "OD9S0za4z", 254 | "version": 2, 255 | "weekStart": "" 256 | } -------------------------------------------------------------------------------- /Utils/scripts/dashboard_templates/ContainerRegistry-1687851648145.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 91, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "grafana-azure-data-explorer-datasource", 34 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 35 | }, 36 | "fieldConfig": { 37 | "defaults": { 38 | "color": { 39 | "mode": "palette-classic" 40 | }, 41 | "custom": { 42 | "align": "auto", 43 | "cellOptions": { 44 | "type": "color-text" 45 | }, 46 | "filterable": true, 47 | "inspect": true 48 | }, 49 | "decimals": 3, 50 | "mappings": [], 51 | "thresholds": { 52 | "mode": "absolute", 53 | "steps": [ 54 | { 55 | "color": "dark-red", 56 | "value": null 57 | }, 58 | { 59 | "color": "dark-green", 60 | "value": 100 61 | } 62 | ] 63 | } 64 | }, 65 | "overrides": [ 66 | { 67 | "matcher": { 68 | "id": "byName", 69 | "options": "id" 70 | }, 71 | "properties": [ 72 | { 73 | "id": "links", 74 | "value": [ 75 | { 76 | "targetBlank": true, 77 | "title": "", 78 | "url": "https://portal.azure.com/${__data.fields.tenantDomain}/resource${__value.raw}/overview" 79 | } 80 | ] 81 | } 82 | ] 83 | } 84 | ] 85 | }, 86 | "gridPos": { 87 | "h": 12, 88 | "w": 24, 89 | "x": 0, 90 | "y": 0 91 | }, 92 | "id": 2, 93 | "options": { 94 | "cellHeight": "sm", 95 | "footer": { 96 | "countRows": false, 97 | "fields": "", 98 | "reducer": [ 99 | "sum" 100 | ], 101 | "show": false 102 | }, 103 | "showHeader": true 104 | }, 105 | "pluginVersion": "9.5.13", 106 | "targets": [ 107 | { 108 | "database": "multi002-metricsdb", 109 | "datasource": { 110 | "type": "grafana-azure-data-explorer-datasource", 111 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 112 | }, 113 | "expression": { 114 | "from": { 115 | "property": { 116 | "name": "Vm_Availability", 117 | "type": "string" 118 | }, 119 | "type": "property" 120 | }, 121 | "groupBy": { 122 | "expressions": [], 123 | "type": "and" 124 | }, 125 | "reduce": { 126 | "expressions": [], 127 | "type": "and" 128 | }, 129 | "where": { 130 | "expressions": [], 131 | "type": "and" 132 | } 133 | }, 134 | "pluginVersion": "4.4.1", 135 | "query": "let v2=Container_Registry_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project component, createdAt, ['date'], id, location, name, successfulPullCount, totalPullCount,\nsolution, subscriptionId, tenantDomain, availability = ((successfulPullCount+successfulPushCount)/(totalPullCount+totalPushCount))*100\n| where ['date'] == datetime($selecteddate)\n and location in ($Region) and subscriptionId in ($Subscriptions)\n and solution in ($Solution) and availability < 100\n| project ['date'] , subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v1=Container_Registry_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project component, createdAt, ['date'], id, location, name, successfulPullCount, totalPullCount,\nsolution, subscriptionId, tenantDomain, availability = ((successfulPullCount+successfulPushCount)/(totalPullCount+totalPushCount))*100\n| where ['date'] == datetime($selecteddate)\n and location in ($Region)\n and solution in ($Solution) and availability < 100\n| project ['date'] , subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v4=Container_Registry_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project component, createdAt, ['date'], id, location, name, successfulPullCount, totalPullCount,\nsolution, subscriptionId, tenantDomain, availability = ((successfulPullCount+successfulPushCount)/(totalPullCount+totalPushCount))*100\n| where ['date'] == datetime($selecteddate)\n and subscriptionId in ($Subscriptions)\n and solution in ($Solution) and availability < 100\n| project ['date'] , subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nlet v3=Container_Registry_Availability\n|join kind=leftouter Subscriptions on subscriptionId\n| project component, createdAt, ['date'], id, location, name, successfulPullCount, totalPullCount,\nsolution, subscriptionId, tenantDomain, availability = ((successfulPullCount+successfulPushCount)/(totalPullCount+totalPushCount))*100\n| where ['date'] == datetime($selecteddate)\n and solution in ($Solution) and availability < 100\n| project ['date'] , subscriptionId, tenantDomain, id, location, name, availability\n| order by availability asc \n| order by ['date'] asc;\nunion kind=outer (v3 | where \"All\" in ($Subscriptions) and \"All\" in ($Region)),\n(v1 | where \"All\" in ($Subscriptions) and \"All\" !in ($Region)),\n(v4 | where \"All\" !in ($Subscriptions) and \"All\" in ($Region)),\n(v2 | where \"All\" !in ($Subscriptions) and \"All\" !in ($Region))", 136 | "querySource": "raw", 137 | "queryType": "KQL", 138 | "rawMode": true, 139 | "refId": "A", 140 | "resultFormat": "table" 141 | } 142 | ], 143 | "title": "# of ContainerRegistries with Availability < 100", 144 | "type": "table" 145 | } 146 | ], 147 | "refresh": "", 148 | "revision": 1, 149 | "schemaVersion": 38, 150 | "style": "dark", 151 | "tags": [], 152 | "templating": { 153 | "list": [ 154 | { 155 | "current": { 156 | "selected": false, 157 | "text": "2023-01-12 00:00:00", 158 | "value": "2023-01-12 00:00:00" 159 | }, 160 | "hide": 2, 161 | "name": "selecteddate", 162 | "options": [ 163 | { 164 | "selected": true, 165 | "text": "2023-01-12 00:00:00", 166 | "value": "2023-01-12 00:00:00" 167 | } 168 | ], 169 | "query": "2023-01-12 00:00:00", 170 | "skipUrlSync": false, 171 | "type": "textbox" 172 | }, 173 | { 174 | "current": { 175 | "isNone": true, 176 | "selected": false, 177 | "text": "None", 178 | "value": "" 179 | }, 180 | "datasource": { 181 | "type": "grafana-azure-data-explorer-datasource", 182 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 183 | }, 184 | "definition": "", 185 | "hide": 2, 186 | "includeAll": false, 187 | "multi": true, 188 | "name": "Region", 189 | "options": [], 190 | "query": "", 191 | "refresh": 1, 192 | "regex": "", 193 | "skipUrlSync": false, 194 | "sort": 0, 195 | "type": "query" 196 | }, 197 | { 198 | "current": { 199 | "isNone": true, 200 | "selected": false, 201 | "text": "None", 202 | "value": "" 203 | }, 204 | "datasource": { 205 | "type": "grafana-azure-data-explorer-datasource", 206 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 207 | }, 208 | "definition": "", 209 | "hide": 2, 210 | "includeAll": false, 211 | "multi": true, 212 | "name": "Subscriptions", 213 | "options": [], 214 | "query": "", 215 | "refresh": 1, 216 | "regex": "", 217 | "skipUrlSync": false, 218 | "sort": 0, 219 | "type": "query" 220 | }, 221 | { 222 | "current": { 223 | "isNone": true, 224 | "selected": false, 225 | "text": "None", 226 | "value": "" 227 | }, 228 | "datasource": { 229 | "type": "grafana-azure-data-explorer-datasource", 230 | "uid": "cdb740a1-7d66-43ba-a012-2ce6e1ee5ea0" 231 | }, 232 | "definition": "", 233 | "hide": 2, 234 | "includeAll": false, 235 | "multi": true, 236 | "name": "Solution", 237 | "options": [], 238 | "query": "", 239 | "refresh": 1, 240 | "regex": "", 241 | "skipUrlSync": false, 242 | "sort": 0, 243 | "type": "query" 244 | } 245 | ] 246 | }, 247 | "time": { 248 | "from": "now-6h", 249 | "to": "now" 250 | }, 251 | "timepicker": {}, 252 | "timezone": "utc", 253 | "title": "ContainerRegistry", 254 | "uid": "mBHFSeX4z", 255 | "version": 2, 256 | "weekStart": "" 257 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Observability at Scale 2 | 3 | This repository contains reference architecture, code sample and dashboard template for tracking Azure resources availability (uptime/downtime) trends. 4 | 5 | This solution implements the pillars of the [Microsoft Azure Well-Architected Framework](https://learn.microsoft.com/en-us/azure/well-architected/), which is a set of guiding tenets that you can use to improve the quality of a workload. One of the key considerations of the solution was the reliability pillar, which ensures that your application can meet the commitments you make to your customers. To support this, this solution helps to ensure that Azure applications are consistently reliable and meet the expectations of customers. In addition, this solution considers the operational excellence pillar, ensuring that processes can keep an application running in production. Together, these pillars come together to ensure that applications remain consistently available and reliable customers. For more information about these framework pillars, see [Overview of the reliability pillar](https://learn.microsoft.com/en-us/azure/well-architected/resiliency/overview) and [Overview of the operational excellence pillar](https://learn.microsoft.com/en-us/azure/well-architected/devops/overview). 6 | 7 | ## Features 8 | 9 | #### Multi-tenant monitoring 10 | 11 | This solultion allows you to track and filter resources across different tenants and subscriptions. Follow the steps in the post-installation section to set this up. 12 | 13 | #### Configurable near real time data pull 14 | 15 | The frequency of availability data pulled can be adjusted down to one minute, or any other desired interval, by updating the MyTimeTrigger environment variable. Follow the steps in the post installation section to modify this value. 16 | 17 | #### Deep linking to Azure Portal 18 | 19 | You can directly navigate to a resource's overview page in the Azure Portal by clicking on the underlined id field in the drill down menu. 20 | 21 | ## Architecture 22 | 23 | The following diagram gives a high-level view of Observability solution. You may download the Visio file from [here](Images/architecture-multi-raw.vsdx) 24 | 25 | ![Solution Architecture](Images/architecturemulti.png) 26 | 27 | Unlike Azure Monitor, which provides the average availability of one resource at a time, this solution provides the average availability of all resources of the same resource type in your subscriptions. For example, instead of providing the availability of one Key Vault, this solution will provide the average availability of all Key Vaults in your subscriptions. 28 | 29 | ## Components 30 | 31 | The above diagram consists of a range of Azure components, which will be further outlined below. 32 | 33 | [**Azure Data Explorer Clusters**](https://learn.microsoft.com/en-us/azure/data-explorer/data-explorer-overview) End-to-end solution for data ingestion, query, visualization, and management. Also used as the time series database for the availability metrics 34 | 35 | [**Resource Graph Explorer**](https://learn.microsoft.com/en-us/azure/governance/resource-graph/overview) Enables running Resource Graph queries directly in the Azure portal. 36 | 37 | [**Service Bus**](https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-messaging-overview) Decouples applications and services from each other, to allow for load-balancing and safe data transfer. 38 | 39 | [**Ingest Function**](https://learn.microsoft.com/en-us/azure/azure-functions/functions-overview?pivots=programming-language-csharp) Loads data records from one or more sources into a table in Azure Data Explorer. Once ingested, the data becomes available for query. 40 | 41 | [**Grafana**](https://learn.microsoft.com/en-us/azure/managed-grafana/overview) Azure managed Grafana to visualize the availability metrics 42 | 43 | [**Azure Blob**](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) Object storage solution for the cloud. Optimized for storing massive amounts of unstructured data. 44 | 45 | [**Key Vault**](https://learn.microsoft.com/en-us/azure/key-vault/general/overview) Cloud-based service that allows you to securely store and manage cryptographic keys, secrets, and certificates used by your applications and services. 46 | 47 | ## Azure Monitor 48 | 49 | This solution calls on the [Azure Monitor Batch API](https://learn.microsoft.com/en-us/rest/api/monitor/metrics-batch/batch?view=rest-monitor-2023-10-01&tabs=HTTP) to pull availability data for multiple resources within a subscription in one call. 50 | 51 | A sample request will look like this 52 | 53 | ``` 54 | 55 | POST "https://{region}.metrics.monitor.azure.com/subscriptions/{subscriptionID}/metrics:getBatch?timespan={timeSpan}&interval=PT1M&metricnames=Availability&aggregation=average&metricNamespace={resourceProvider}&autoadjusttimegrain=true&api-version=2023-03-01-preview" 56 | 57 | ``` 58 | 59 | With multiple resource IDs passed in the body of the request 60 | 61 | ``` 62 | 63 | { 64 | "resourceids": [ 65 | "/subscriptions/12345678-abcd-1234-abcd-123456789abc/resourceGroups/TestGroup/providers/Microsoft.Storage/storageAccounts/TestStorage1", 66 | "/subscriptions/12345678-abcd-1234-abcd-123456789abc/resourceGroups/TestGroup/providers/Microsoft.Storage/storageAccounts/TestStorage2" 67 | ] 68 | } 69 | 70 | ``` 71 | 72 | ## Recommended SKU 73 | 74 | The recommended SKU for this Kusto cluster is Standard_E8ads_v5. You can monitor this and scale up as needed by checking the application insights for the TimerStartPipelineFunction. You may see some 429 Kusto errors, meaning that your requests are being rate limited. The requests will wait some and be retried, so you should not experience data loss. However, it is best to scale up if you are seeing these errors to avoid further issues. 75 | 76 | Additionally, ensure that your tenant does not have any policies in place that would prevent Terraform from creating a client SP secret, or you will see an error in the deployment. 77 | 78 | ## Availability Metrics 79 | 80 | In Azure services, availability refers to the percentage of time that a service or application is available and functioning as expected. 81 | 82 | The following availability metrics are supported by Azure Monitor. This version of the solution queries only these metrics. 83 | 84 | | Resource Type | Metric Name(Azure Monitor) | Availability metric calculation | 85 | |--------------- |---------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | 86 | | AKS Server Node | [kube_node_status_condition](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftcontainerservicemanagedclusters) | (Ready / (Ready + Not Ready)) x 100 | 87 | | Load Balancer | [VipAvailability](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftnetworkloadbalancers) | - | 88 | | Firewall | [FirewallHealth](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftnetworkazurefirewalls) | - | 89 | | Storage | [Availability](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftclassicstoragestorageaccounts) | - | 90 | | Cosmos DB | [ServiceAvailability](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftdocumentdbdatabaseaccounts) | - | 91 | | Key Vault | [Availability](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported#microsoftkeyvaultvaults) | - | 92 | | Event Hubs | [IncomingRequests, ServerErrors](https://learn.microsoft.com/en-us/azure/event-hubs/monitor-event-hubs-reference) | ((IncomingRequests - ServerErrors) / IncomingRequests) x 100 | 93 | | Container Registry | [Successful/Total Push, Successful/Total Pull](https://learn.microsoft.com/en-us/azure/container-registry/monitor-service-reference) | ((Successful Push + Pull)/(Total Push + Pull)) x 100 | 94 | | Log Analytics | [AvailabilityRate_Query](https://learn.microsoft.com/en-us/azure/azure-monitor/reference/supported-metrics/microsoft-operationalinsights-workspaces-metrics) | - | 95 | 96 | ## Visualization 97 | 98 | In this section, you will see how the Grafana dashboard displays availability metrics over a given time frame for each queried resource type. 99 | 100 | ![Solution Visualization](Images/MultiTenantVisualization.PNG) 101 | 102 | You can also drill down to a more detailed view of the monitored resources and click on the ID field to navigate to the resource's overview page in Azure Portal. 103 | 104 | ![Drill Down Screen](Images/DrillDown1.PNG) 105 | 106 | ![Drill Down Screen 2](Images/drilldown2.png) 107 | 108 | 109 | 110 | ## Getting Started 111 | 112 | The following section describes the Prerequisites and Installation steps to deploy the solution. 113 | 114 | ### Prerequisites 115 | 116 | > Note: Azure Role Permissions: User should have access to create ManagedIdentity/Service Principal on the subscription 117 | 118 | #### Environment 119 | 120 | The script can be executed in Linux - Ubuntu 20.04 (VM, WSL). 121 | 122 | > Note: currently Azure Cloud Shell is not supported since it uses az-cli > 2.46.0 123 | 124 | ### Install using Terraform 125 | 126 | ```bash 127 | ## Clone git repo into the folder 128 | repolink="" 129 | codePath=$"./observability" 130 | git clone $repolink $codePath 131 | 132 | ## Please setup the following required parameters for the script to run: 133 | ## prefix - prefix string to identify the resources created with this deployment. eg: test 134 | ## subscriptionId - subscriptionId where the solution will be deployed to 135 | ## location - location where the azure resources will be created. eg: eastus 136 | 137 | # change directory to where the repo is cloned 138 | cd $codePath 139 | 140 | # set current working directory 141 | currentDir=$(pwd) 142 | 143 | # install pre-requisites 144 | bash $currentDir/Utils/scripts/pre-requisites.sh 145 | 146 | #downgrade az-cli to use version < 2.46 147 | apt-cache policy azure-cli 148 | sudo apt-get install azure-cli=-1~ 149 | eg: sudo apt-get install azure-cli=2.46.0-1~focal (Codename - focal/bionic/bullseye etc) 150 | 151 | # change directory to where Terraform main.tf is located 152 | cd $currentDir/Utils/scripts/Terraform 153 | 154 | ``` 155 | 156 | > Note: if you are deploying feature improvements on top of an existing deployment, please copy over the tfstate files from the folders resources,grafana-datasource and grafana-dashboards from your existing deployment to the cloned repository 157 | 158 | ![terraform-folders](Images/terraform-folders.png) 159 | 160 | ```bash 161 | 162 | #log in to the tenant where the subscription to host the resources is present 163 | az login 164 | 165 | #list the subscriptions under the tenant 166 | az account show 167 | 168 | #set the subscription where the resources are to be deployed 169 | az account set --subscription 170 | 171 | ## 1. Create resources using Terraform 172 | cd resources 173 | 174 | #initialize terraform providers 175 | terraform init 176 | 177 | # run a plan on the root file 178 | terraform plan -var="prefix=" -var="subscriptionId=" -var="location=" -parallelism= 179 | eg: terraform plan -var="prefix=test" -var="subscriptionId=00000000-0000-0000-0000-000000000000" -var="location=eastus" -parallelism=1 180 | 181 | # run apply on the root file 182 | terraform apply -var="prefix=" -var="subscriptionId=" -var="location=" -parallelism= 183 | eg: terraform apply -var="prefix=test" -var="subscriptionId=00000000-0000-0000-0000-000000000000" -var="location=eastus" -parallelism=1 184 | note: make sure to confirm resource creation with a "yes" when the prompt appears on running this command 185 | 186 | # add "grafana admin" role to the user as described here - https://learn.microsoft.com/en-us/azure/managed-grafana/how-to-share-grafana-workspace?tabs=azure-portal 187 | 188 | # run post installation script to set up some additional variables 189 | sh post_install.sh 190 | 191 | # create api key and export all variables 192 | export TF_VAR_database_name=$(terraform output -raw database_name) 193 | export TF_VAR_cluster_url=$(terraform output -raw cluster_url) 194 | export TF_VAR_sp_object_id=$(terraform output -raw sp_object_id) 195 | export TF_VAR_prefix=$(terraform output -raw prefix) 196 | export TF_VAR_url=$(az grafana show -g $TF_VAR_prefix-RG -n $TF_VAR_prefix-grafana -o json | jq -r .properties.endpoint) 197 | export TF_VAR_token=$(az grafana api-key create --key `date +%s` --name $TF_VAR_prefix-grafana -g $TF_VAR_prefix-RG -r editor --time-to-live 60m -o json | jq -r .key) 198 | 199 | ## 2. Update grafana instance to create datasource, folders and dashboards using Terraform 200 | cd ../grafana-datasource 201 | 202 | #initialize terraform providers 203 | terraform init -upgrade 204 | 205 | # run a plan on the root file 206 | terraform plan 207 | 208 | # run apply on the root file 209 | terraform apply 210 | 211 | ## 3. Update grafana instance to create folders and dashboards using Terraform 212 | cd ../grafana-dashboards 213 | 214 | #initialize terraform providers 215 | terraform init -upgrade 216 | 217 | # run a plan on the root file 218 | terraform plan 219 | 220 | # run apply on the root file 221 | terraform apply 222 | ``` 223 | 224 | ### Post Installation 225 | 226 | #### Post Installation Steps 227 | 228 | The solution relies on the following data to be present in the "Resource Providers" and "Subscriptions" tables before it can be used to visualize the data. Follow the steps below to complete the post installation steps. 229 | 230 | #### Updating Resource Types 231 | 232 | 1. Download the file - [ResourceTypes.csv](Utils/scripts/csv_Import/ResourceTypes.csv) to insert the list of resource providers to be monitored in the Resource_Providers table. 233 | 234 | ![githubfiledownload](Images/githubfiledownload-1.png) 235 | > Note: While saving to local ensure that you save the file with csv extension, the default is set to .txt 236 | 237 | 2. Data ingestion: follow the steps described in the [link](DATAINGESTION.md) to complete the data ingestion 238 | 239 | #### Updating Subscriptions 240 | 241 | 1. Download the file - [subscriptions.csv](Utils/scripts/csv_Import/subscriptions.csv) to local 242 | 2. Modify the CSV to include details of the subscriptions and tenants for which you want to track resource health. 243 | 3. Follow the data ingestion steps as outlined in the previous instructions for ResourceType.csv file. 244 | 245 | Finally, add "Monitoring Reader" role for the Managed Identity and Service Principal created by script to the subscriptions that you want to monitor within the tenant where you have deployed the solution. 246 | 247 | #### Enabling ingestion to ADX with MSI 248 | 249 | Currently, the following command needs to be executed manually on the ADX cluster to enable native ingestion from storage with MSI. 250 | ``` 251 | .alter-merge cluster policy managed_identity "[{ 'ObjectId' : '%%%%', 'AllowedUsages' : 'NativeIngestion' }]" 252 | ``` 253 | The ObjectId of the ADX system-assigned identity should be inputted here. This ObjectId can be found by navigating to the ADX Cluster > Security + Networking > Identity in the Azure portal. 254 | 255 | #### Monitoring Additional Tenants 256 | In order to support monitoring of additional tenants, you will have add the appropriate service principal credentials to Key Vault. Follow the steps below to create and upload the client secrets. 257 | 258 | 1. Creating a Service Principal: follow the steps described to [create a multitenant app registration and client secret](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app) in the tenant you would like to monitor. 259 | 3. Add the "Monitoring Reader" role for this Service Principal to any subscriptions which you would like to monitor within the tenant. 260 | 4. Upload the Service Principal credentials as a secret to Key Vault in the following format: 261 | 262 | Secret Name: tenant-[TenantId] 263 | 264 | Secret Value: {"ClientId":"[ServicePrincipalClientId]","ClientSecret":"[ServicePrincipalSecretValue]"} 265 | 266 | #### Configuring near real-time monitoring 267 | In order to update the frequency of pulling availability metrics, you can navigate to the app settings of the TimerStartPipelineFunction. Here, you can modify the MyTimeTrigger variable in environment variables as seen below to reduce to 1 minute, or your desired interval. 268 | 269 | ![Time Trigger](Images/mytimetrigger.png) 270 | 271 | 272 | #### Grafana access 273 | 274 | To add other users to view/edit the Grafana dashboard, follow [adding role assignment to managed grafana](https://learn.microsoft.com/en-us/azure/managed-grafana/how-to-share-grafana-workspace?tabs=azure-portal) 275 | 276 | #### Storage access 277 | 278 | sas token - expires in a year need to update it 279 | 280 | #### az grafana known issue with higher az cli versions 281 | 282 | az grafana create not compatible with az cli versions > 2.46 ongoing issue - [https://github.com/Azure/azure-cli-extensions/issues/6221](https://github.com/Azure/azure-cli-extensions/issues/6221), advice to use lower 283 | versions of cli <=2.46 until the issue is resolved. 284 | 285 | ![recommended cli version](Images/az-cli-version.png) 286 | 287 | #### persisting tfstate files 288 | 289 | please ensure you are storing the tfstate files in the following locations so that they can be used to deploy further improvements in the future 290 | 291 | ![terraform-folders](Images/terraform-folders.png) 292 | 293 | #### Incremental Deployment on existing solution 294 | 295 | Note: for MSFT Tenant, remove the secret in key vault in your existing deployment before incremental deployment, and save it(save name and secret value). Add it back to key vault manually after incremental deployment is finished. 296 | 297 | 1. clone the new branch 298 | 2. go to /mcsa-observability/Utils/scripts/Terraform/resources of the exisitng deployed branch and copy the terraform.tfstate file and paste over to the same directory of the new undeployed branch 299 | 3. follow through all the steps of previous deployment instruction -------------------------------------------------------------------------------- /Utils/scripts/table_scripts.kql: -------------------------------------------------------------------------------- 1 | //create raw tables 2 | .create-merge table Aksservernode_Availability_Raw (metrics: dynamic) with (folder = "Raw") 3 | 4 | .create-merge table Cosmosdb_Availability_Raw (metrics: dynamic) with (folder = "Raw") 5 | 6 | .create-merge table Firewall_Availability_Raw (metrics: dynamic) with (folder = "Raw") 7 | 8 | .create-merge table Keyvault_Availability_Raw (metrics: dynamic) with (folder = "Raw") 9 | 10 | .create-merge table Loadbalancer_Availability_Raw (metrics: dynamic) with (folder = "Raw") 11 | 12 | .create-merge table Storage_Availability_Raw (metrics: dynamic) with (folder = "Raw") 13 | 14 | .create-merge table Eventhubs_Availability_Raw (metrics: dynamic) with (folder = "Raw") 15 | 16 | .create-merge table Container_Registry_Availability_Raw (metrics: dynamic) with (folder = "Raw") 17 | 18 | .create-merge table LogAnalytics_Availability_Raw (metrics: dynamic) with (folder = "Raw") 19 | 20 | // Create ingestion mapping 21 | .create-or-alter table Aksservernode_Availability_Raw ingestion json mapping 'RawMetricsMapping' '[{"Column":"metrics","Properties":{"path":"$"}}]' 22 | 23 | .create-or-alter table Cosmosdb_Availability_Raw ingestion json mapping 'RawMetricsMapping' '[{"Column":"metrics","Properties":{"path":"$"}}]' 24 | 25 | .create-or-alter table Firewall_Availability_Raw ingestion json mapping 'RawMetricsMapping' '[{"Column":"metrics","Properties":{"path":"$"}}]' 26 | 27 | .create-or-alter table Keyvault_Availability_Raw ingestion json mapping 'RawMetricsMapping' '[{"Column":"metrics","Properties":{"path":"$"}}]' 28 | 29 | .create-or-alter table Loadbalancer_Availability_Raw ingestion json mapping 'RawMetricsMapping' '[{"Column":"metrics","Properties":{"path":"$"}}]' 30 | 31 | .create-or-alter table Storage_Availability_Raw ingestion json mapping 'RawMetricsMapping' '[{"Column":"metrics","Properties":{"path":"$"}}]' 32 | 33 | .create-or-alter table Eventhubs_Availability_Raw ingestion json mapping 'RawMetricsMapping' '[{"Column":"metrics","Properties":{"path":"$"}}]' 34 | 35 | .create-or-alter table Container_Registry_Availability_Raw ingestion json mapping 'RawMetricsMapping' '[{"Column":"metrics","Properties":{"path":"$"}}]' 36 | 37 | .create-or-alter table LogAnalytics_Availability_Raw ingestion json mapping 'RawMetricsMapping' '[{"Column":"metrics","Properties":{"path":"$"}}]' 38 | 39 | //create adx tables 40 | .create-merge table Resource_Providers (name: string, ['type']: string, resultTableName: string) 41 | 42 | .create-merge table Subscription_Names (subscriptionId: guid, name: string) 43 | 44 | .create-merge table Subscriptions (solution: string, tenancy: string, component: string, tenantId:guid, tenantDomain:string, subscriptionId: guid, createdAt: datetime) 45 | 46 | .create-merge table Subscriptions_Processed (subscriptionId: guid, dateProcessed: datetime) 47 | 48 | .create-merge table Aksservernode_Availability (['date']: datetime, ['id']: string, nodeReady: decimal, nodeNotReady: decimal, nodeUnknown: decimal, location: string, subscriptionId: guid, name: string) 49 | 50 | .create-merge table Cosmosdb_Availability (['date']: datetime, name: string, availability: decimal, subscriptionId: guid, location: string, ['id']: string) 51 | 52 | .create-merge table Firewall_Availability (['date']: datetime, ['id']: string, availability: decimal, subscriptionId: guid, location: string, name: string) 53 | 54 | .create-merge table Keyvault_Availability (['date']: datetime, ['id']: string, availability: decimal, subscriptionId: guid, location: string, name: string) 55 | 56 | .create-merge table Loadbalancer_Availability (['date']: datetime, ['id']: string, availability: decimal, subscriptionId: guid, location: string, name: string) 57 | 58 | .create-merge table Storage_Availability (['date']: datetime, ['id']: string, availability: decimal, subscriptionId: guid, location: string, name: string) 59 | 60 | .create-merge table Eventhubs_Availability (['date']: datetime, ['id']: string, incomingRequests: decimal, serverErrors: decimal, location: string, subscriptionId: guid, name: string) 61 | 62 | .create-merge table Container_Registry_Availability (['date']: datetime, ['id']: string, successfulPullCount: decimal, totalPullCount: decimal, successfulPushCount: decimal, totalPushCount: decimal, location: string, subscriptionId: guid, name: string) 63 | 64 | .create-merge table LogAnalytics_Availability (['date']: datetime, ['id']: string, availability: decimal, subscriptionId: guid, location: string, name: string) 65 | 66 | //create functions 67 | 68 | .create-or-alter function Parse_Eventhubs_Availability() { 69 | let requests = Eventhubs_Availability_Raw 70 | | mv-expand values = metrics.values 71 | | extend id = tostring(values.resourceid) 72 | | extend subscriptionId = split(id, '/')[2] 73 | | extend name = split(id, '/')[-1] 74 | | extend region = tostring(values.resourceregion) 75 | | mv-expand value = values.value 76 | | where tostring(value.name.value) == 'IncomingRequests' 77 | | mv-expand timeseries = value.timeseries 78 | | mv-expand data = timeseries.data 79 | | project timestamp = todatetime(data.timeStamp), id = tostring(id), incomingrequests = todecimal(data.average), 80 | location = tostring(region), subscriptionId = toguid(subscriptionId), name = tostring(name); 81 | let errors = Eventhubs_Availability_Raw 82 | | mv-expand values = metrics.values 83 | | extend id = tostring(values.resourceid) 84 | | extend subscriptionId = split(id, '/')[2] 85 | | extend name = split(id, '/')[-1] 86 | | extend region = tostring(values.resourceregion) 87 | | mv-expand value = values.value 88 | | where tostring(value.name.value) == 'ServerErrors' 89 | | mv-expand timeseries = value.timeseries 90 | | mv-expand data = timeseries.data 91 | | project timestamp = todatetime(data.timeStamp), id = tostring(id), servererrors = todecimal(data.average), 92 | location = tostring(region), subscriptionId = toguid(subscriptionId), name = tostring(name); 93 | (requests | join kind=leftouter errors on $left.timestamp == $right.timestamp and $left.id == $right.id) 94 | | project timestamp, id, incomingrequests, servererrors, location, subscriptionId, name 95 | } 96 | 97 | .create-or-alter function Parse_Container_Registry_Availability() { 98 | let successfulPull = Container_Registry_Availability_Raw 99 | | mv-expand values = metrics.values 100 | | extend id = tostring(values.resourceid) 101 | | extend subscriptionId = split(id, '/')[2] 102 | | extend name = split(id, '/')[-1] 103 | | extend region = tostring(values.resourceregion) 104 | | mv-expand value = values.value 105 | | where tostring(value.name.value) == 'SuccessfulPullCount' 106 | | mv-expand timeseries = value.timeseries 107 | | mv-expand data = timeseries.data 108 | | distinct timestamp = todatetime(data.timeStamp), id = tostring(id), successfulpullcount = todecimal(data.average), 109 | location = tostring(region), subscriptionId = toguid(subscriptionId), name = tostring(name); 110 | let totalPull = Container_Registry_Availability_Raw 111 | | mv-expand values = metrics.values 112 | | extend id = tostring(values.resourceid) 113 | | extend subscriptionId = split(id, '/')[2] 114 | | extend name = split(id, '/')[-1] 115 | | extend region = tostring(values.resourceregion) 116 | | mv-expand value = values.value 117 | | where tostring(value.name.value) == 'TotalPullCount' 118 | | mv-expand timeseries = value.timeseries 119 | | mv-expand data = timeseries.data 120 | | distinct timestamp = todatetime(data.timeStamp), id = tostring(id), totalpullcount = todecimal(data.average), 121 | location = tostring(region), subscriptionId = toguid(subscriptionId), name = tostring(name); 122 | let successfulPush = Container_Registry_Availability_Raw 123 | | mv-expand values = metrics.values 124 | | extend id = tostring(values.resourceid) 125 | | extend subscriptionId = split(id, '/')[2] 126 | | extend name = split(id, '/')[-1] 127 | | extend region = tostring(values.resourceregion) 128 | | mv-expand value = values.value 129 | | where tostring(value.name.value) == 'SuccessfulPushCount' 130 | | mv-expand timeseries = value.timeseries 131 | | mv-expand data = timeseries.data 132 | | distinct timestamp = todatetime(data.timeStamp), id = tostring(id), successfulpushcount = todecimal(data.average), 133 | location = tostring(region), subscriptionId = toguid(subscriptionId), name = tostring(name); 134 | let totalPush = Container_Registry_Availability_Raw 135 | | mv-expand values = metrics.values 136 | | extend id = tostring(values.resourceid) 137 | | extend subscriptionId = split(id, '/')[2] 138 | | extend name = split(id, '/')[-1] 139 | | extend region = tostring(values.resourceregion) 140 | | mv-expand value = values.value 141 | | where tostring(value.name.value) == 'TotalPushCount' 142 | | mv-expand timeseries = value.timeseries 143 | | mv-expand data = timeseries.data 144 | | distinct timestamp = todatetime(data.timeStamp), id = tostring(id), totalpushcount = todecimal(data.average), 145 | location = tostring(region), subscriptionId = toguid(subscriptionId), name = tostring(name); 146 | (successfulPull | join kind = innerunique totalPull on $left.timestamp == $right.timestamp and $left.id == $right.id) 147 | | join kind = innerunique successfulPush on $left.timestamp == $right.timestamp and $left.id == $right.id 148 | | join kind = innerunique totalPush on $left.timestamp == $right.timestamp and $left.id == $right.id 149 | | project timestamp, id, successfulpullcount, totalpullcount, successfulpushcount, totalpushcount, location, subscriptionId, name 150 | } 151 | 152 | .create-or-alter function Parse_Aksservernode_Availability() { 153 | let ready = Aksservernode_Availability_Raw 154 | | mv-expand values = metrics.values 155 | | extend id = tostring(values.resourceid) 156 | | extend subscriptionId = split(id, '/')[2] 157 | | extend name = split(id, '/')[-1] 158 | | extend region = tostring(values.resourceregion) 159 | | mv-expand value = values.value 160 | | where tostring(value.name.value) == 'kube_node_status_condition' 161 | | mv-expand timeseries = value.timeseries 162 | | mv-expand metadatavalues = timeseries.metadatavalues 163 | | where tostring(metadatavalues.name.value) == 'status2' and tostring(metadatavalues.value) == 'Ready' 164 | | mv-expand data = timeseries.data 165 | | project timestamp = todatetime(data.timeStamp), id = tostring(id), nodeReady = todecimal(data.average), location = tostring(region), subscriptionId = toguid(subscriptionId), name = tostring(name); 166 | let notReady = Aksservernode_Availability_Raw 167 | | mv-expand values = metrics.values 168 | | extend id = tostring(values.resourceid) 169 | | extend subscriptionId = split(id, '/')[2] 170 | | extend name = split(id, '/')[-1] 171 | | extend region = tostring(values.resourceregion) 172 | | mv-expand value = values.value 173 | | where tostring(value.name.value) == 'kube_node_status_condition' 174 | | mv-expand timeseries = value.timeseries 175 | | mv-expand metadatavalues = timeseries.metadatavalues 176 | | where tostring(metadatavalues.name.value) == 'status2' and tostring(metadatavalues.value) == 'NotReady' 177 | | mv-expand data = timeseries.data 178 | | project timestamp = todatetime(data.timeStamp), id = tostring(id), nodeNotReady = todecimal(data.average), location = tostring(region), subscriptionId = toguid(subscriptionId), name = tostring(name); 179 | let unknown = Aksservernode_Availability_Raw 180 | | mv-expand values = metrics.values 181 | | extend id = tostring(values.resourceid) 182 | | extend subscriptionId = split(id, '/')[2] 183 | | extend name = split(id, '/')[-1] 184 | | extend region = tostring(values.resourceregion) 185 | | mv-expand value = values.value 186 | | where tostring(value.name.value) == 'kube_node_status_condition' 187 | | mv-expand timeseries = value.timeseries 188 | | mv-expand metadatavalues = timeseries.metadatavalues 189 | | where tostring(metadatavalues.name.value) == 'status2' and tostring(metadatavalues.value) == 'unknown' 190 | | mv-expand data = timeseries.data 191 | | project timestamp = todatetime(data.timeStamp), id = tostring(id), nodeUnknown = todecimal(data.average), location = tostring(region), subscriptionId = toguid(subscriptionId), name = tostring(name); 192 | (ready | join kind=leftouter notReady on $left.timestamp == $right.timestamp and $left.id == $right.id) 193 | | join kind=leftouter unknown on $left.timestamp == $right.timestamp and $left.id == $right.id 194 | | project timestamp, id, nodeReady, nodeNotReady, nodeUnknown, location, subscriptionId, name 195 | } 196 | 197 | .create-or-alter function Parse_Cosmosdb_Availability() { 198 | Cosmosdb_Availability_Raw 199 | | mv-expand values = metrics.values 200 | | extend id = tostring(values.resourceid) 201 | | extend subscriptionId = split(id, '/')[2] 202 | | extend name = split(id, '/')[-1] 203 | | extend region = tostring(values.resourceregion) 204 | | mv-expand value = values.value 205 | | where tostring(value.name.value) == 'ServiceAvailability' 206 | | mv-expand timeseries = value.timeseries 207 | | mv-expand data = timeseries.data 208 | | project 209 | timestamp = todatetime(data.timeStamp), 210 | name = tostring(name), 211 | availability = todecimal(data.average), 212 | subscriptionId = toguid(subscriptionId), 213 | location = tostring(region), 214 | id = tostring(id) 215 | } 216 | 217 | .create-or-alter function Parse_Firewall_Availability() { 218 | Firewall_Availability_Raw 219 | | mv-expand values = metrics.values 220 | | extend id = tostring(values.resourceid) 221 | | extend subscriptionId = split(id, '/')[2] 222 | | extend name = split(id, '/')[-1] 223 | | extend region = tostring(values.resourceregion) 224 | | mv-expand value = values.value 225 | | where tostring(value.name.value) == 'FirewallHealth' 226 | | mv-expand timeseries = value.timeseries 227 | | mv-expand data = timeseries.data 228 | | project 229 | timestamp = todatetime(data.timeStamp), 230 | id = tostring(id), 231 | availability = todecimal(data.average), 232 | subscriptionId = toguid(subscriptionId), 233 | location = tostring(region), 234 | name = tostring(name) 235 | } 236 | 237 | .create-or-alter function Parse_Keyvault_Availability() { 238 | Keyvault_Availability_Raw 239 | | mv-expand values = metrics.values 240 | | extend id = tostring(values.resourceid) 241 | | extend subscriptionId = split(id, '/')[2] 242 | | extend name = split(id, '/')[-1] 243 | | extend region = tostring(values.resourceregion) 244 | | mv-expand value = values.value 245 | | where tostring(value.name.value) == 'Availability' 246 | | mv-expand timeseries = value.timeseries 247 | | mv-expand data = timeseries.data 248 | | project 249 | timestamp = todatetime(data.timeStamp), 250 | id = tostring(id), 251 | availability = todecimal(data.average), 252 | subscriptionId = toguid(subscriptionId), 253 | location = tostring(region), 254 | name = tostring(name) 255 | } 256 | 257 | .create-or-alter function Parse_Loadbalancer_Availability() { 258 | Loadbalancer_Availability_Raw 259 | | mv-expand values = metrics.values 260 | | extend id = tostring(values.resourceid) 261 | | extend subscriptionId = split(id, '/')[2] 262 | | extend name = split(id, '/')[-1] 263 | | extend region = tostring(values.resourceregion) 264 | | mv-expand value = values.value 265 | | where tostring(value.name.value) == 'VipAvailability' 266 | | mv-expand timeseries = value.timeseries 267 | | mv-expand data = timeseries.data 268 | | project 269 | timestamp = todatetime(data.timeStamp), 270 | id = tostring(id), 271 | availability = todecimal(data.average), 272 | subscriptionId = toguid(subscriptionId), 273 | location = tostring(region), 274 | name = tostring(name) 275 | } 276 | 277 | .create-or-alter function Parse_Storage_Availability() { 278 | Storage_Availability_Raw 279 | | mv-expand values = metrics.values 280 | | extend id = tostring(values.resourceid) 281 | | extend subscriptionId = split(id, '/')[2] 282 | | extend name = split(id, '/')[-1] 283 | | extend region = tostring(values.resourceregion) 284 | | mv-expand value = values.value 285 | | where tostring(value.name.value) == 'Availability' 286 | | mv-expand timeseries = value.timeseries 287 | | mv-expand data = timeseries.data 288 | | project 289 | timestamp = todatetime(data.timeStamp), 290 | id = tostring(id), 291 | availability = todecimal(data.average), 292 | subscriptionId = toguid(subscriptionId), 293 | location = tostring(region), 294 | name = tostring(name) 295 | } 296 | 297 | .create-or-alter function Parse_LogAnalytics_Availability() { 298 | LogAnalytics_Availability_Raw 299 | | mv-expand values = metrics.values 300 | | extend id = tostring(values.resourceid) 301 | | extend subscriptionId = split(id, '/')[2] 302 | | extend name = split(id, '/')[-1] 303 | | extend region = tostring(values.resourceregion) 304 | | mv-expand value = values.value 305 | | where tostring(value.name.value) == 'AvailabilityRate_Query' 306 | | mv-expand timeseries = value.timeseries 307 | | mv-expand data = timeseries.data 308 | | project 309 | timestamp = todatetime(data.timeStamp), 310 | id = tostring(id), 311 | availability = todecimal(data.average), 312 | subscriptionId = toguid(subscriptionId), 313 | location = tostring(region), 314 | name = tostring(name) 315 | } 316 | 317 | // Update results table policies 318 | .alter table Aksservernode_Availability policy update @'[{"Source": "Aksservernode_Availability_Raw", "Query": "Parse_Aksservernode_Availability()", "IsEnabled": "True", "IsTransactional": true}]' 319 | 320 | .alter table Cosmosdb_Availability policy update @'[{"Source": "Cosmosdb_Availability_Raw", "Query": "Parse_Cosmosdb_Availability()", "IsEnabled": "True", "IsTransactional": true}]' 321 | 322 | .alter table Firewall_Availability policy update @'[{"Source": "Firewall_Availability_Raw", "Query": "Parse_Firewall_Availability()", "IsEnabled": "True", "IsTransactional": true}]' 323 | 324 | .alter table Keyvault_Availability policy update @'[{"Source": "Keyvault_Availability_Raw", "Query": "Parse_Keyvault_Availability()", "IsEnabled": "True", "IsTransactional": true}]' 325 | 326 | .alter table Loadbalancer_Availability policy update @'[{"Source": "Loadbalancer_Availability_Raw", "Query": "Parse_Loadbalancer_Availability()", "IsEnabled": "True", "IsTransactional": true}]' 327 | 328 | .alter table Storage_Availability policy update @'[{"Source": "Storage_Availability_Raw", "Query": "Parse_Storage_Availability()", "IsEnabled": "True", "IsTransactional": true}]' 329 | 330 | .alter table Eventhubs_Availability policy update @'[{"Source": "Eventhubs_Availability_Raw", "Query": "Parse_Eventhubs_Availability()", "IsEnabled": "True", "IsTransactional": true}]' 331 | 332 | .alter table Container_Registry_Availability policy update @'[{"Source": "Container_Registry_Availability_Raw", "Query": "Parse_Container_Registry_Availability()", "IsEnabled": "True", "IsTransactional": true}]' 333 | 334 | .alter table LogAnalytics_Availability policy update @'[{"Source": "LogAnalytics_Availability_Raw", "Query": "Parse_LogAnalytics_Availability()", "IsEnabled": "True", "IsTransactional": true}]' 335 | 336 | 337 | // Adding zero retention policies on Raw tables 338 | .alter-merge table Aksservernode_Availability_Raw policy retention softdelete = 0s 339 | 340 | .alter-merge table Cosmosdb_Availability_Raw policy retention softdelete = 0s 341 | 342 | .alter-merge table Firewall_Availability_Raw policy retention softdelete = 0s 343 | 344 | .alter-merge table Keyvault_Availability_Raw policy retention softdelete = 0s 345 | 346 | .alter-merge table Loadbalancer_Availability_Raw policy retention softdelete = 0s 347 | 348 | .alter-merge table Storage_Availability_Raw policy retention softdelete = 0s 349 | 350 | .alter-merge table Eventhubs_Availability_Raw policy retention softdelete = 0s 351 | 352 | .alter-merge table Container_Registry_Availability_Raw policy retention softdelete = 0s 353 | 354 | .alter-merge table LogAnalytics_Availability_Raw policy retention softdelete = 0s 355 | 356 | 357 | // Adding default (100 years) policy to Subs, RPs, and Sub Names tables 358 | .alter table Subscriptions policy retention "{}" 359 | 360 | .alter table Resource_Providers policy retention "{}" 361 | 362 | .alter table Subscription_Names policy retention "{}" --------------------------------------------------------------------------------