├── ADF ├── ADFPerformance.pbit ├── ADF_PipelinesandActivitiesJoined.kql ├── completed_activities_over_given_time_period.kql ├── completed_pipelines_over_given_time_period.kql └── dimension_pipeline_history.kql ├── AS ├── AAS Metrics and Diagnostics.pbit ├── AAS_Metrics.kql ├── DistinctUsersPerHour.kql ├── LongRunningQueries.kql ├── MostCommonlyRunQueries.kql ├── ProcessingReport.pbit ├── ProcessingTime.kql ├── ReplicaSyncTimings.kql ├── aas_findaquery.kql ├── as_querytimings_fe_vs_se_breakdown.kql ├── errors_per_resource.kql ├── heavy_qpu_queries.kql ├── processing_perpartition.kql ├── reports_hitting_aas.kql └── ssas_directquery_timings.kql ├── Automation ├── JobStreamOutputs.kql └── RunbookHistory.kql ├── DataLake ├── allRequestEvents.kql ├── numberofCallsandTypesAgainstContainers.kql ├── numberofOpsByIdentity.kql └── throttling.kql ├── IOTHub ├── DeviceConnections.kql └── DeviceIdentityOperations.kql ├── README.md ├── Storage ├── NumberOfOperationsByStatusCode.kql ├── NumberOfOperationsByStorageContainerAccessedAndOperation.kql ├── NumberOfOperationsByUserAgent ├── azStorageMetrics.pbit ├── powershell_getloganalytics_uri.ps1 └── showStorageMetrics.kql └── Synapse ├── Count of All Queued Queries.kql ├── LongSessions.kql ├── Most Active Resource Classes.kql ├── SessionDurations.kql ├── ShowRequestStepsForASpecificQuery.kql ├── TimingsPerSQLDistribution.kql ├── Top Requests Most Impacted by Data Movement.kql ├── completed_user_queries_over_given_timespan_v2.kql └── exec_requests.kql /ADF/ADFPerformance.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christophermschmidt/monitor/2e32d9e3730eceeb07c40fdbdf7e2f966ce5e848/ADF/ADFPerformance.pbit -------------------------------------------------------------------------------- /ADF/ADF_PipelinesandActivitiesJoined.kql: -------------------------------------------------------------------------------- 1 | let adfdata = //start by getting the runs that completed, either by using the Completed or Failed status codes 2 | union AzureDiagnostics, workspace("").AzureDiagnostics 3 | //only pull the last 7 days 4 | | where TimeGenerated >= ago(7d) 5 | //where the service is Data Factory 6 | | where ResourceProvider == "MICROSOFT.DATAFACTORY"; 7 | adfdata 8 | //only grab pipeline runs 9 | | where Category == "PipelineRuns" 10 | | where status_s == "Succeeded" or status_s == "Failed" 11 | | project runId_g, pipelineName_s, Message, start_t, end_t, Data_Provider_Key= Parameters_PartitionName_s , TimePeriodEndDate = Parameters_SubpartitionName_s, Target_Table = Parameters_TrgtTableName_s, status_s, Resource 12 | | extend pDurationMs = datetime_diff('millisecond', end_t, start_t) 13 | | extend pipelineRunId_g=runId_g 14 | | where pipelineName_s == "pipeline_name" 15 | | join kind = leftouter ( 16 | adfdata 17 | //only grab activity runs 18 | | where Category == "ActivityRuns" 19 | | where status_s == "Succeeded" or status_s == "Failed" 20 | | project Resource, pipelineRunId_g, activityName_s, astart_t=start_t, aend_t=end_t, activityType_s, effectiveIntegrationRuntime_s, Output_executionDuration_d, Output_durationInQueue_integrationRuntimeQueue_d, Input_storedProcedureName_s, Input_storedProcedureParameters_at_enabled_value_s, Input_storedProcedureParameters_load_mode_value_s, DataProviderKey = Input_storedProcedureParameters_part_col1_val_value_s, TimePeriodEndDate = Input_storedProcedureParameters_part_col2_val_value_s, SourceTable = Input_storedProcedureParameters_src_table_value_s, SubjectArea = Input_storedProcedureParameters_subject_area_value_s 21 | | extend aDurationMs = datetime_diff('millisecond', aend_t, astart_t) 22 | ) on pipelineRunId_g 23 | | project runId_g, pipelineName_s, Message, start_t, end_t, Data_Provider_Key, pDurationMs, aDurationMs, TimePeriodEndDate, Target_Table, status_s, Resource, activityName_s, astart_t, aend_t, activityType_s, Output_durationInQueue_integrationRuntimeQueue_d, Output_executionDuration_d, Input_storedProcedureName_s, Input_storedProcedureParameters_at_enabled_value_s, Input_storedProcedureParameters_load_mode_value_s, SubjectArea 24 | | summarize avg(aDurationMs) by Resource, activityName_s, activityType_s 25 | | order by activityType_s 26 | -------------------------------------------------------------------------------- /ADF/completed_activities_over_given_time_period.kql: -------------------------------------------------------------------------------- 1 | //start by getting the runs that completed, either by using the Completed or Failed status codes 2 | AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.DATAFACTORY" 4 | | where Category == "ActivityRuns" 5 | | where status_s == "Succeeded" or status_s == "Failed" 6 | | project pipelineRunId_g, activityName_s, start_t, end_t, activityType_s, effectiveIntegrationRuntime_s, Output_executionDuration_d, Output_durationInQueue_integrationRuntimeQueue_d, Input_storedProcedureName_s, Input_storedProcedureParameters_at_enabled_value_s, Input_storedProcedureParameters_load_mode_value_s, DataProviderKey = Input_storedProcedureParameters_part_col1_val_value_s, TimePeriodEndDate = Input_storedProcedureParameters_part_col2_val_value_s, SourceTable = Input_storedProcedureParameters_src_table_value_s, SubjectArea = Input_storedProcedureParameters_subject_area_value_s 7 | | extend DurationMs = datetime_diff('millisecond', end_t, start_t), Duration_Min = datetime_diff('minute', end_t, start_t) 8 | -------------------------------------------------------------------------------- /ADF/completed_pipelines_over_given_time_period.kql: -------------------------------------------------------------------------------- 1 | //start by getting the runs that completed, either by using the Completed or Failed status codes 2 | AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.DATAFACTORY" 4 | | where Category == "PipelineRuns" 5 | | where status_s == "Succeeded" or status_s == "Failed" 6 | | project runId_g, pipelineName_s, Message, start_t, end_t, Data_Provider_Key= Parameters_PartitionName_s , TimePeriodEndDate = Parameters_SubpartitionName_s, Target_Table = Parameters_TrgtTableName_s 7 | | extend DurationMs = datetime_diff('millisecond', end_t, start_t) -------------------------------------------------------------------------------- /ADF/dimension_pipeline_history.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.DATAFACTORY" 3 | | where Category == "PipelineRuns" 4 | | where status_s == "Succeeded" or status_s == "Failed" 5 | | project Resource, runId_g, pipelineName_s, Message, start_t, end_t, Annotations_s 6 | | extend DurationMs = datetime_diff('millisecond', end_t, start_t) 7 | | where pipelineName_s contains "dim" -------------------------------------------------------------------------------- /AS/AAS Metrics and Diagnostics.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christophermschmidt/monitor/2e32d9e3730eceeb07c40fdbdf7e2f966ce5e848/AS/AAS Metrics and Diagnostics.pbit -------------------------------------------------------------------------------- /AS/AAS_Metrics.kql: -------------------------------------------------------------------------------- 1 | //gather the LogMetrics for the AAS instance out of the AzureDiagnostics table. This would include any replicas if they have been created for the instance. 2 | AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" 4 | | where OperationName == "LogMetric" -------------------------------------------------------------------------------- /AS/DistinctUsersPerHour.kql: -------------------------------------------------------------------------------- 1 | // the below query returns the number of distinct users which connected to each instance (including replicas) in 1 hour buckets over the timespan specified 2 | let window = AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES"; 4 | window 5 | | where OperationName has "QueryEnd" and EventSubclass_s has "3" 6 | | summarize dcount(EffectiveUsername_s) by ServerName_s, timebin=bin(StartTime_t, 1h) 7 | | order by ServerName_s, timebin -------------------------------------------------------------------------------- /AS/LongRunningQueries.kql: -------------------------------------------------------------------------------- 1 | //Use the below query to find/identify queries that exceeded a number of milliseconds when running. Update the last line to the number of milliseconds you want to filter for. The default value is 10,000 (10 seconds) 2 | AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" 4 | | where OperationName == "QueryEnd" 5 | | project TimeGenerated, DatabaseName_s, StartTime_t, EndTime_t, Duration_s, CPUTime_s, EffectiveUsername_s, TextData_s, RootActivityId_g 6 | | where toint(Duration_s) >= 10000 -------------------------------------------------------------------------------- /AS/MostCommonlyRunQueries.kql: -------------------------------------------------------------------------------- 1 | //use the below query to see the most commonly run queries from Power BI. The below reutnrs the number of users who ran the query, the total number of times run, and average/min/max duration. It is filtered to only return the top 50 results by default. 2 | let window = AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" and ApplicationName_s == "PowerBI" and OperationName == "QueryEnd" 4 | | where TimeGenerated >= ago(7d); 5 | window 6 | // convert the Duration column to int 7 | | extend Duration_i = toint(Duration_s) 8 | //filter out errors 9 | | where Error_s == "0" 10 | //group by the queries and count, max, min, and avg them 11 | | summarize avg(Duration_i), min(Duration_i), max(Duration_i), count(), NumUsersWhoRanQuery = dcount(EffectiveUsername_s) by TextData_s 12 | //only keep queries that had a max duration longer than 5 seconds over the given timespan 13 | | where max_Duration_i >= 5000 14 | //keep the top 50 of the most commonly run queries used by the most users 15 | | top 50 by NumUsersWhoRanQuery desc -------------------------------------------------------------------------------- /AS/ProcessingReport.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christophermschmidt/monitor/2e32d9e3730eceeb07c40fdbdf7e2f966ce5e848/AS/ProcessingReport.pbit -------------------------------------------------------------------------------- /AS/ProcessingTime.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" 3 | | where EventClass_s == "COMMAND_END" 4 | | extend DurationMs=extract(@"([^,]*)", 1,Duration_s, typeof(long)) 5 | | project RootActivityId_g, TextData_s, StartTime_t, EndTime_t 6 | | where TextData_s contains "Refresh" 7 | | where StartTime_t <> todatetime('1601-01-01T00:00:00Z') -------------------------------------------------------------------------------- /AS/ReplicaSyncTimings.kql: -------------------------------------------------------------------------------- 1 | //use the below query to identify the amount of time used to perform a replica sync across the nodes. This query is designed to work when the replicasyncmode is set to 1 (default) 2 | AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" 4 | //event subclass 41 executes the initial detach against the primary :rw node. 42 executes the attach against the replicas 5 | | where OperationName == "ProgressReportEnd" and EventSubclass_s in (41,42) 6 | | project duration_ms = Duration_s ,duration_sec = datetime_diff('second', EndTime_t, StartTime_t), StartTime = StartTime_t , EndTime = EndTime_t , CPUTime_ms = toint(CPUTime_s) , Success_s , Error_s , RootActivityId_g , IsQueryScaleoutReadonlyInstance_s , DatabaseName = DatabaseName_s , Server = ServerName_s , ResourceGroup , Instance = Resource 7 | //if you want the sync time rolled up per instance, use the below summarize statement. if you want the individualized replica times, comment the below line out 8 | | summarize total_duration_in_seconds = sum(duration_sec), StartTime = min(StartTime), EndTime = max(EndTime), Success_ind = min(Success_s), sum(CPUTime_ms) by RootActivityId_g, Instance, DatabaseName 9 | | order by StartTime asc -------------------------------------------------------------------------------- /AS/aas_findaquery.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" 3 | | where OperationName == "QueryEnd" 4 | | where TextData_s contains "" 5 | 6 | //once you find the query, run this query and put in the RAID 7 | AzureDiagnostics 8 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" 9 | | where RootActivityId_g == "" -------------------------------------------------------------------------------- /AS/as_querytimings_fe_vs_se_breakdown.kql: -------------------------------------------------------------------------------- 1 | let window = AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" //and RootActivityId_g == ""; 3 | window 4 | | where OperationName has "QueryEnd" or (OperationName has "CommandEnd" and EventSubclass_s == 38) 5 | | where extract(@"([^,]*)", 1,Duration_s, typeof(long)) > 0 6 | | extend DurationMs=extract(@"([^,]*)", 1,Duration_s, typeof(long)) 7 | | extend Engine_CPUTime=extract(@"([^,]*)", 1,CPUTime_s, typeof(long)) 8 | | project OperationName,RootActivityId_g ,TextData_s,DatabaseName_s,ApplicationName_s,StartTime_t,EndTime_t,Duration_s , EffectiveUsername_s ,User_s,EventSubclass_s ,DurationMs,Engine_CPUTime, ServerName_s , IsQueryScaleoutReadonlyInstance_s 9 | | join kind=leftouter ( 10 | window 11 | | where OperationName == "ProgressReportEnd" or (OperationName == "VertiPaqSEQueryEnd" and EventSubclass_s != 10) or OperationName == "DiscoverEnd" or (OperationName has "CommandEnd" and EventSubclass_s != 38) 12 | | summarize sum_Engine_CPUTime = sum(extract(@"([^,]*)", 1,CPUTime_s, typeof(long))) by RootActivityId_g 13 | ) on RootActivityId_g 14 | | extend totalCPU = sum_Engine_CPUTime + Engine_CPUTime 15 | //| summarize sum(totalCPU) by 16 | //OperationName,RootActivityId_g ,TextData_s,DatabaseName_s,ApplicationName_s,StartTime_t,EndTime_t,Duration_s , EffectiveUsername_s ,User_s,EventSubclass_s ,DurationMs,Engine_CPUTime,sum_Engine_CPUTime,totalCPU 17 | | order by DurationMs desc 18 | | project totalCPU, FormulaEngineCPUTime = Engine_CPUTime, StorageEngineCPUTime = sum_Engine_CPUTime, Duration_s -------------------------------------------------------------------------------- /AS/errors_per_resource.kql: -------------------------------------------------------------------------------- 1 | //# of errors per resource 2 | AzureDiagnostics 3 | | where Status_s == "Failed" or Level == "Error" or ResultType contains "Fail" 4 | | extend consolidated_status = coalesce(Status_s, Level, ResultType) 5 | | summarize count() by Resource, consolidated_status -------------------------------------------------------------------------------- /AS/heavy_qpu_queries.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" and Resource contains "" 3 | | where OperationName != "VertiPaqSEQueryEnd" or EventSubclass_s == "0" 4 | //| where ServerName_s contains "-" 5 | //calculate the amount of time spent in the engine 6 | | extend Engine_CPUTime=extract(@"([^,]*)", 1,CPUTime_s, typeof(long)) 7 | //summarize the QPU and Duration metrics and grab the query text by each request (RootActivityId) 8 | | summarize cpu = sum(Engine_CPUTime)/1000, TimeInSeconds = max(toint(Duration_s))/1000, max(iif(OperationName == "QueryBegin", TextData_s, "")) by RootActivityId_g 9 | //order by the most expensive query 10 | | order by cpu desc 11 | 12 | //use this query to narrow in on a heavier qpu query from above. 13 | AzureDiagnostics 14 | | where RootActivityId_g == "" 15 | //| where OperationName == "QueryEnd" 16 | | project OperationName, CPUTime_s, TextData_s, EventSubclass_s, CurrentTime_t, Duration_s 17 | | order by CurrentTime_t asc 18 | 19 | AzureDiagnostics 20 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" and ServerName_s contains "" 21 | | where OperationName != "VertiPaqSEQueryEnd" or EventSubclass_s == "0" 22 | | where extract(@"([^,]*)", 1,Duration_s, typeof(long)) > 0 23 | //| where ServerName_s contains "-" 24 | | extend Engine_CPUTime=extract(@"([^,]*)", 1,CPUTime_s, typeof(long)) 25 | | summarize qpu = sum(Engine_CPUTime)/3000/4 by bin(StartTime_t, 1s) 26 | | render timechart 27 | -------------------------------------------------------------------------------- /AS/processing_perpartition.kql: -------------------------------------------------------------------------------- 1 | //use the below query to identify the processing time per partition in the model. The DurationMs report the number of milliseconds that elapsed in order to perform the processing. If you want to see just the overall timing per partition, //use the top one. If you want to see a detailed breakdown per partition showing the amount of time spent running the SQL query, transferring data over the network, and then compressing the data, use the 2nd one. 2 | AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" 4 | //capture progress reort end events and event subclass 59. 5 | | where EventClass_s == "PROGRESS_REPORT_END" and EventSubclass_s == "59" and TextData_s startswith "Finished processing partition" 6 | | extend DurationMs=extract(@"([^,]*)", 1,Duration_s, typeof(long)) 7 | | summarize count() by OperationName, EventClass_s, EventSubclass_s, DurationMs, ObjectName_s, TextData_s, TimeGenerated 8 | | order by DurationMs 9 | 10 | 11 | 12 | AzureDiagnostics 13 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" 14 | //capture progress reort end events and event subclass 59. 15 | | where EventClass_s == "PROGRESS_REPORT_END" and ((EventSubclass_s == 17 and TextData_s contains "reading") or (EventSubclass_s == 25 and TextData_s contains "SELECT") or (EventSubclass_s == 59 and TextData_s contains "partition") or EventSubclass_s == 44 or EventSubclass_s == 6) 16 | | extend DurationMs=extract(@"([^,]*)", 1,Duration_s, typeof(long)) 17 | | summarize count() by OperationName, EventClass_s, EventSubclass_s, DurationMs, ObjectName_s, TextData_s, TimeGenerated 18 | | order by ObjectName_s, DurationMs -------------------------------------------------------------------------------- /AS/reports_hitting_aas.kql: -------------------------------------------------------------------------------- 1 | 2 | //This query finds the number of times queries were run per report with max and average durations for each over a given timespan 3 | AzureDiagnostics 4 | | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" 5 | | where OperationName == "QueryEnd" 6 | | extend ReportName = extractjson("$.Sources[0].ReportId", ApplicationContext_s) 7 | | project ApplicationContext_s, toint(Duration_s), ReportName, Resource, ServerName_s, TextData_s, EffectiveUsername_s, RootActivityId_g, ApplicationName_s 8 | | summarize NumberOfQueriesRun = dcount(RootActivityId_g), DistinctUsers=dcount(EffectiveUsername_s), ReportDurationMax = max(Duration_s), ReportDurationAvg = avg(Duration_s) by ApplicationName_s, Resource, ReportName -------------------------------------------------------------------------------- /AS/ssas_directquery_timings.kql: -------------------------------------------------------------------------------- 1 | let window = AzureDiagnostics | where ResourceProvider == "MICROSOFT.ANALYSISSERVICES" and ServerName_s contains "nameofaasserver"; 2 | window 3 | | where OperationName == "DirectQueryBegin" 4 | | project OperationName,TextData_s,DatabaseName_s,ApplicationName_s,StartTime_t,EffectiveUsername_s ,User_s, ServerName_s, CPUTime_s, SPID_s, RootActivityId_g 5 | | join kind=leftouter ( 6 | window 7 | | where OperationName == "DirectQueryEnd" 8 | | project OperationName,TextData_s, EndTime_t, Duration_s, RootActivityId_g 9 | ) on RootActivityId_g 10 | | project ApplicationName_s, StartTime_t, EndTime_t, EffectiveUsername_s , TextData_s , CPUTime_s , Duration_s, RootActivityId_g 11 | -------------------------------------------------------------------------------- /Automation/JobStreamOutputs.kql: -------------------------------------------------------------------------------- 1 | //azure powershell job stream outputs from the last 14 days 2 | AzureDiagnostics 3 | | where TimeGenerated >= startofday(ago(14d)) 4 | | where ResourceProvider == "MICROSOFT.AUTOMATION" and Category == "JobStreams" 5 | | project-away ResourceId, Tenant_g, Computer, ResourceGroup, SubscriptionId, ResourceProvider, Resource, ResourceType, Type 6 | | sort by TimeGenerated desc -------------------------------------------------------------------------------- /Automation/RunbookHistory.kql: -------------------------------------------------------------------------------- 1 | let window = AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.AUTOMATION" 3 | | where Category == "JobLogs"; 4 | window 5 | | where ResultType == "Started" 6 | | project CorrelationId, RunbookName_s, JobId_g , ResourceGroup, Resource, StartTime = TimeGenerated 7 | | join kind=leftouter ( 8 | window 9 | | where ResultType == "Completed" 10 | | project CorrelationId, EndTime = TimeGenerated 11 | ) on CorrelationId 12 | | order by StartTime desc -------------------------------------------------------------------------------- /DataLake/allRequestEvents.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.DATALAKESTORE" and TimeGenerated >= ago(3d) 3 | //request logs capture every API request made on the Data Lake Storage Gen1 account 4 | | where Category == "Requests" 5 | | project Resource, ResourceGroup, ResourceType, OperationName, ResultType, CorrelationId, HttpMethod_s, Path_s, identity_s, UserId_g, StoreEgressSize_d, StoreIngressSize_d, CallerIPAddress, StartTime_t, EndTime_t, RequestDuration = datetime_diff("Millisecond", EndTime_t, StartTime_t) 6 | | extend Path = split(Path_s, '/') 7 | | mv-expand root = Path[0], level1 = Path[1], level2 = Path[2], level3 = Path[3], level4 = Path[4], level5 = Path[5], level6 = Path[6], level7 = Path[7], level8 = Path[8], level9 = Path[9] 8 | | extend likelyfileName = coalesce(level9, level8, level7, level6, level5, level4) -------------------------------------------------------------------------------- /DataLake/numberofCallsandTypesAgainstContainers.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.DATALAKESTORE" and TimeGenerated >= ago(3d) 3 | //request logs capture every API request made on the Data Lake Storage Gen1 account 4 | | where Category == "Requests" 5 | | project Resource, ResourceGroup, ResourceType, OperationName, ResultType, CorrelationId, HttpMethod_s, Path_s, identity_s, UserId_g, StoreEgressSize_d, StoreIngressSize_d, CallerIPAddress, StartTime_t, EndTime_t, RequestDuration = datetime_diff("Millisecond", EndTime_t, StartTime_t) 6 | | extend Path = split(Path_s, '/') 7 | | mv-expand root = Path[0], level1 = Path[1], level2 = Path[2], level3 = Path[3], level4 = Path[4], level5 = Path[5], level6 = Path[6], level7 = Path[7], level8 = Path[8], level9 = Path[9] 8 | | summarize count() by tostring(level3), HttpMethod_s -------------------------------------------------------------------------------- /DataLake/numberofOpsByIdentity.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.DATALAKESTORE" and TimeGenerated >= ago(3d) 3 | | where Category == "Requests" 4 | | project identity_s, CorrelationId, HttpMethod_s 5 | | summarize count() by identity_s, HttpMethod_s -------------------------------------------------------------------------------- /DataLake/throttling.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.DATALAKESTORE" and TimeGenerated >= ago(3d) 3 | | where Category == "Requests" 4 | //am I being throttled? Have I submitted too many requests within a given timeframe? 5 | | where ResultType == 429 6 | | project Resource, ResourceGroup, ResourceType, OperationName, ResultType, CorrelationId, HttpMethod_s, Path_s, identity_s, UserId_g, StoreEgressSize_d, StoreIngressSize_d, CallerIPAddress, StartTime_t, EndTime_t -------------------------------------------------------------------------------- /IOTHub/DeviceConnections.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where Category == "Connections" 3 | and OperationName == "deviceConnect" -------------------------------------------------------------------------------- /IOTHub/DeviceIdentityOperations.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where Category == "DeviceIdentityOperations" -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # monitor 2 | Open Log Analytics queries and samples on querying different Azure resources and services. Includes sample Power BI reports 3 | 4 | use this to gain starter queries and Power Bi Report Samples for monitoring different services through Log Analytics and Power BI 5 | -------------------------------------------------------------------------------- /Storage/NumberOfOperationsByStatusCode.kql: -------------------------------------------------------------------------------- 1 | //show the count of all operations over a gievn timespan, bucketed by 5 minutes. dispaly the result as a timechart 2 | StorageBlobLogs 3 | | where TimeGenerated >= ago (3d) 4 | | project TimeGenerated, OperationName, AccountName, Protocol, AuthenticationType, StatusCode, StatusText, DurationMs, ServerLatencyMs, Uri, UserAgentHeader, Category 5 | | summarize count() by StatusCode, StatusText 6 | 7 | //then use this query to narrow in on something of note: 8 | StorageBlobLogs 9 | | where TimeGenerated >= ago (3d) 10 | | project TimeGenerated, OperationName, AccountName, Protocol, AuthenticationType, StatusCode, StatusText, DurationMs, ServerLatencyMs, Uri, UserAgentHeader, Category 11 | | where StatusCode == "500" 12 | 13 | //you can use this result to go find out more information about the client that initiated the request. was it doing anything unusual during the timeframe? -------------------------------------------------------------------------------- /Storage/NumberOfOperationsByStorageContainerAccessedAndOperation.kql: -------------------------------------------------------------------------------- 1 | //show the number of operations by the storage container that was accessed and the UserAgent that accessed the container by minute over the given timespan 2 | StorageBlobLogs 3 | | where TimeGenerated >= ago (3d) 4 | | project TimeGenerated, OperationName, AccountName, Protocol, AuthenticationType, StatusCode, StatusText, DurationMs, ServerLatencyMs, Uri, UserAgentHeader, Category 5 | | extend Path = split(Uri, '/') 6 | | mv-expand root = Path[0], level1 = Path[1], level2 = Path[2], level3 = Path[3], level4 = Path[4], level5 = Path[5], level6 = Path[6], level7 = Path[7], level8 = Path[8], level9 = Path[9] 7 | | summarize count() by tostring(level3), bin(TimeGenerated, 1m), UserAgentHeader 8 | | order by TimeGenerated -------------------------------------------------------------------------------- /Storage/NumberOfOperationsByUserAgent: -------------------------------------------------------------------------------- 1 | The user agent normally passes along information about where the request is coming from. By using this we can determine where heavier workloads are appearing. 2 | StorageBlobLogs 3 | | where TimeGenerated >= ago (3d) 4 | | project TimeGenerated, UserAgentHeader 5 | | summarize count() by UserAgentHeader, bin(TimeGenerated, 5m) 6 | | order by TimeGenerated 7 | | render timechart -------------------------------------------------------------------------------- /Storage/azStorageMetrics.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/christophermschmidt/monitor/2e32d9e3730eceeb07c40fdbdf7e2f966ce5e848/Storage/azStorageMetrics.pbit -------------------------------------------------------------------------------- /Storage/powershell_getloganalytics_uri.ps1: -------------------------------------------------------------------------------- 1 | #import-module Az 2 | 3 | #login to azure 4 | Connect-AzAccount 5 | 6 | Install-Module Az.OperationalInsights 7 | 8 | $resourceGroupName = "" 9 | $workspaceName = "" 10 | 11 | #$uri = "https://portal.loganalytics.io/subscriptions/{0}/resourcegroups/{1}/workspaces/{2}" -f $subscriptionId, $resourceGroupName, $workspaceName 12 | 13 | #instead of the uri, get the guid for the power bi report instead: 14 | $workspace = Get-AzOperationalInsightsWorkspace -ResourceGroupName $resourceGroupName -Name $workspaceName 15 | $GuidString = $workspace.CustomerId.Guid.ToString() 16 | $pbistring = "https://api.loganalytics.io/v1/workspaces/$GuidString/query" 17 | $pbistring 18 | 19 | 20 | -------------------------------------------------------------------------------- /Storage/showStorageMetrics.kql: -------------------------------------------------------------------------------- 1 | //the azure metrics table within log analytics logs high level metrics within a specific service over a given timespan 2 | AzureMetrics 3 | | where TimeGenerated <= ago(7d) 4 | | where ResourceProvider == "MICROSOFT.STORAGE" 5 | | project TimeGenerated, MetricName, Maximum, Minimum, Average, TimeGrain, UnitName 6 | 7 | //for example, if you want to see over a specific metric what the activity looked like here is a sample: 8 | AzureMetrics 9 | | where Resource == "" 10 | | project TimeGenerated, Resource, MetricName, Total, TimeGrain, UnitName 11 | | where MetricName == "Transactions" 12 | | summarize max(Total) by Resource, MetricName, UnitName, bin(TimeGenerated, 1m) 13 | | order by TimeGenerated 14 | | render timechart -------------------------------------------------------------------------------- /Synapse/Count of All Queued Queries.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where Category contains "waits" 3 | | where Type_s == "UserConcurrencyResourceType" 4 | -------------------------------------------------------------------------------- /Synapse/LongSessions.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where TimeGenerated >= ago(3d) 3 | | where ResourceProvider == "MICROSOFT.SQL" 4 | | where Category == "ExecRequests" 5 | | summarize EntryPoint = min(SubmitTime_t), ExitPoint = max(EndTime_t) by SessionId_s 6 | | extend TotalSessionLength = datetime_diff('minute',ExitPoint, EntryPoint) 7 | | where TotalSessionLength >= 5 8 | | order by TotalSessionLength desc -------------------------------------------------------------------------------- /Synapse/Most Active Resource Classes.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where Category contains "ExecRequests" 3 | | where Status_s == "Completed" 4 | | summarize totalQueries = dcount(RequestId_s) by ResourceClass_s 5 | -------------------------------------------------------------------------------- /Synapse/SessionDurations.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where TimeGenerated >= ago(3d) 3 | | where ResourceProvider == "MICROSOFT.SQL" 4 | | where Category == "ExecRequests" 5 | | summarize max(ResourceClass_s), EntryPoint = min(SubmitTime_t), ExitPoint = max(EndTime_t) by SessionId_s 6 | | extend TotalSessionLength = datetime_diff('minute',ExitPoint, EntryPoint) 7 | | where TotalSessionLength >= 5 8 | | order by TotalSessionLength desc 9 | -------------------------------------------------------------------------------- /Synapse/ShowRequestStepsForASpecificQuery.kql: -------------------------------------------------------------------------------- 1 | //how many rows were returned from a specific query? 2 | AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.SQL" 4 | | where RequestId_s == "QID71279970" and Category == "RequestSteps" //SqlRequests 5 | | where Status_s == "Complete" -------------------------------------------------------------------------------- /Synapse/TimingsPerSQLDistribution.kql: -------------------------------------------------------------------------------- 1 | //show the run time per sql pool distribution 2 | AzureDiagnostics 3 | | where ResourceProvider == "MICROSOFT.SQL" 4 | | where RequestId_s == "QID71279970" and Category == "SqlRequests" 5 | //only grab completed steps. Match the StepIndex to the appropriate StepIndex from the RequestSteps query previously run 6 | | where Status_s == "Complete" and StepIndex_d == 0 7 | | distinct TimeGenerated, Resource, RequestId_s, DistributionId_d, Status_s, StartTime_t, EndTime_t, Spid_d, Command_s 8 | | extend RunTime = datetime_diff('millisecond', EndTime_t, StartTime_t) 9 | | order by RunTime desc 10 | -------------------------------------------------------------------------------- /Synapse/Top Requests Most Impacted by Data Movement.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where Category == "RequestSteps" 3 | | where OperationType_s in ("ShuffleMoveOperation", "BroadcastMoveOperation", "PartitionMoveOperation", "RoundRobinMoveOperation", "SingleSourceRoundRobinMoveOperation", "MoveOperation", "TrimMoveOperation") 4 | | where Status_s == "Complete" 5 | | project RequestId_s, duration=datetime_diff('millisecond',EndTime_t, StartTime_t) 6 | | order by duration desc 7 | -------------------------------------------------------------------------------- /Synapse/completed_user_queries_over_given_timespan_v2.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where ResourceProvider == "MICROSOFT.SQL" and Resource == "ResourceNameHere" 3 | | where Category == "ExecRequests" //and Status_s == "Completed" 4 | //| where RequestId_s == "QID7352418" 5 | | summarize ResourceClass = max(ResourceClass_s), Command = max(Command_s), ExplainOutput = max(ExplainOutput_s), TimeGenerated = min(TimeGenerated), SubmitTime = min(SubmitTime_t), EndCompileTime = max(EndCompileTime_t), StartTime = max(StartTime_t), EndTime = max(EndTime_t) by SessionId_s, RequestId_s 6 | | extend CleanCompileTime = format_datetime(EndCompileTime, "MM-dd-yyyy h:m:s.fffffff") 7 | | extend Duration = datetime_diff('millisecond', EndTime , SubmitTime ), CompileTime = datetime_diff('millisecond',EndCompileTime, SubmitTime), RunTime = datetime_diff('millisecond',EndTime, EndCompileTime ) 8 | -------------------------------------------------------------------------------- /Synapse/exec_requests.kql: -------------------------------------------------------------------------------- 1 | AzureDiagnostics 2 | | where Resource == "MSAPACTESTONLY" and Category == "ExecRequests" 3 | | project TimeGenerated, SessionID_s, SubmitTime_t, EndCompileTime_t, StartTime_t, EndTime_t , Label_s , StatementType_s, status_s, Command_s, Category, OperationName, RequestId_s --------------------------------------------------------------------------------