├── LICENSE ├── queries.kusto └── README.md /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 tobmcv 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /queries.kusto: -------------------------------------------------------------------------------- 1 | // Search across all datasets 2 | search "event name" | take 10 3 | 4 | // Searching in a single dataset 5 | Perf | search "Contoso" 6 | 7 | // Searching in multiple datasets 8 | search in (Perf, Event, Alert) "Contoso" | take 10 9 | 10 | // Searching in specific columns for a match 11 | Perf 12 | | search CounterName=="Available MBytes" | take 10 13 | 14 | //Searching for a term anywhere in text 15 | Perf 16 | | search "*Bytes*" | take 10 17 | 18 | // Searching for columns that begins with 19 | Perf 20 | | search * startswith "Bytes" | take 10 21 | 22 | // Searching for columns that ends with 23 | Perf 24 | | search * endswith "Bytes" | take 10 25 | 26 | //Search starts with specific term, anything in between and ends with a specific term 27 | Perf 28 | | search "Free*bytes" | take 10 29 | 30 | 31 | // **Combining searches** 32 | // Multiple terms 33 | Perf 34 | | search "Free*bytes" and ("C:" or "D:") | take 10 35 | 36 | // Using regular expressions 37 | Perf 38 | | search InstanceName matches regex "[A-Z]:" | take 10 39 | 40 | 41 | // **Time and timerange** 42 | //Dynamic timerange 43 | | where timestamp between (datetime(2019-01-01T00:01:24.615Z)..now()) 44 | 45 | 46 | // **Where** 47 | Perf 48 | | where TimeGenerated >= ago(1h) 49 | 50 | //Note the time range for the query is automatically set in the query when we use time operators in our where clause 51 | //Using the `and` statement 52 | Perf 53 | | where TimeGenerated >= ago(1h) and CounterName == "Bytes Received/sec" | take 10 54 | 55 | 56 | // Using the `or` statement 57 | Perf 58 | | where TimeGenerated >= ago(1h) and (CounterName == "Bytes Received/sec" or CounterName == "% Processor Time") | take 10 59 | 60 | 61 | //Stacking multiple where clauses 62 | Perf 63 | | where TimeGenerated >= ago(1h) 64 | | where (CounterName == "Bytes Received/sec" 65 | or 66 | CounterName == "% Processor Time") 67 | | where CounterValue > 0 68 | | take 10 69 | 70 | 71 | //Other uses for where - simulating search with where 72 | Perf 73 | | where * has "Bytes" | take 10 74 | 75 | 76 | //**Positional matching** with where 77 | // Matches words beginning with Bytes 78 | Perf 79 | | where * hasprefix "Bytes" | take 10 80 | 81 | 82 | //Matches words ending with Bytes 83 | Perf 84 | | where * hassuffix "Bytes" | take 10 85 | 86 | 87 | //Using `contains` 88 | Perf 89 | | where * contains "Bytes" | take 10 90 | 91 | //Using regular expressions with where 92 | Perf 93 | | where InstanceName matches regex "[A-Z]:" | take 10 94 | 95 | 96 | //**Take and Limit command** 97 | //Take retrieves a random number of rows, f.ex 98 | Perf 99 | | take 10 100 | 101 | //Takes 10 rows 102 | 103 | //Limit is the equivalent 104 | Perf 105 | | limit 10 106 | 107 | // **Count operator** 108 | // Count rows in a table 109 | Perf 110 | | count 111 | // Counts all rows in the Perf dataset 112 | 113 | // Count can be used with filtering to count rows in a selected table 114 | Perf 115 | | where TimeGenerated >= ago(1h) 116 | and CounterName == "Bytes received/sec" 117 | and CounterValue > 0 118 | | count 119 | 120 | 121 | // **Summarize** 122 | // Aggregation function for summarizing data in a table, can include `by` 123 | Perf 124 | | summarize count() by CounterName 125 | 126 | 127 | // Summarizing multiple columns 128 | Perf 129 | | summarize count() by ObjectName, CounterName 130 | 131 | // Note it generates a third column called `count_`. The name is automatically generated. 132 | 133 | // If we want to supply a more suitable name as a variable, we can name it 134 | Perf 135 | | summarize PerfCount=count() 136 | by ObjectName, CounterName 137 | 138 | 139 | // With Summarize, use other aggregation functions 140 | Perf 141 | | where CounterName == "% Free Space" 142 | | summarize NumberOfEntries=count() 143 | , AverageFreeSpace=avg(CounterValue) 144 | by CounterName 145 | 146 | 147 | // Using `Bin` to create logical groups 148 | Perf 149 | | summarize NumberOfEntries=count() 150 | by bin(TimeGenerated, 1d) 151 | 152 | 153 | // Using other values for binning 154 | Perf 155 | | where CounterName == "% Free Space" 156 | | summarize NumberOfRowsAtThisPercentLevel=count() 157 | by bin(CounterValue, 10) 158 | 159 | 160 | 161 | // **Extend** 162 | // Extend allows you to create calculated columns to add to your tables 163 | Perf 164 | | where CounterName == "Free Megabytes" 165 | | extend FreeGB = CounterValue / 1000 166 | 167 | 168 | // You can also create multiple columns 169 | Perf 170 | | where CounterName == "Free Megabytes" 171 | | extend FreeGB = CounterValue / 1000 172 | , FreeKB = CounterValue * 1000 173 | 174 | 175 | // Repeating a column 176 | Perf 177 | | where CounterName == "Free Megabytes" 178 | | extend FreeGB = CounterValue / 1000 179 | , FreeMB = CounterValue 180 | , FreeKB = CounterValue * 1000 181 | 182 | 183 | // Create new string values 184 | // We can create a new column with string values 185 | Perf 186 | | where TimeGenerated >= ago(10m) 187 | | extend ObjectCounter = strcat(ObjectName, " - ", CounterName) 188 | 189 | // We use `strcat` to concatenate strings 190 | 191 | // **Project command** 192 | 193 | // Project allows us to select which columns we want in our table 194 | Perf 195 | | project ObjectName 196 | , CounterName 197 | , InstanceName 198 | , CounterValue 199 | , TimeGenerated 200 | 201 | 202 | // Project and Extend are very useful when creating tables with our specific data. 203 | Perf 204 | | project ObjectName 205 | , CounterName 206 | , InstanceName 207 | , CounterValue 208 | , TimeGenerated 209 | | extend FreeGB = CounterValue / 1000 210 | , FreeMB = CounterValue 211 | , FreeKB = CounterValue * 1000 212 | 213 | 214 | // If we want to omit a specific column we must calculate our values first, then project afterwards 215 | Perf 216 | | extend FreeGB = CounterValue / 1000 217 | , FreeMB = CounterValue 218 | , FreeKB = CounterValue * 1000 219 | | project ObjectName 220 | , CounterName 221 | , InstanceName 222 | , TimeGenerated 223 | , FreeGB 224 | , FreeMB 225 | , FreeKB 226 | 227 | // OR we can just project our data and calculate it on the fly 228 | Perf 229 | | where CounterName == "Free Megabytes" 230 | | project ObjectName 231 | , CounterName 232 | , InstanceName 233 | , TimeGenerated 234 | , FreeGB = CounterValue / 1000 235 | , FreeMB = CounterValue 236 | , FreeKB = CounterValue * 1000 237 | 238 | // `Project-away` lets you remove selected columns from the output 239 | Perf 240 | | where CounterName == "Free Megabytes" 241 | | project-away TenantId 242 | , SourceSystem 243 | , CounterPath 244 | , MG 245 | 246 | 247 | // `Project-rename` lets you rename a specific column 248 | Perf 249 | | where TimeGenerated > ago(1h) 250 | | project-rename myRenamedComputer = Computer 251 | 252 | 253 | // **Distinct** 254 | // Retrieves a list of deduplicated values 255 | Perf 256 | | distinct ObjectName, CounterName 257 | 258 | 259 | // Only show unique error events 260 | Event 261 | | where EventLevelName == "Error" 262 | | distinct Source 263 | 264 | //Using top for the first 20 rows 265 | Perf 266 | | top 20 by TimeGenerated desc 267 | 268 | // Sort rows ascending with `asc` or descending with `desc` 269 | 270 | // Combining everything so far 271 | Perf 272 | | where CounterName == "Free Megabytes" // Get the free Megabytes 273 | and TimeGenerated >= ago(1h) // ... within the last hour 274 | | project Computer // For each return the Computer Name 275 | , TimeGenerated // ...and when the counter was generated 276 | , CounterName // ...and the Counter Name 277 | , FreeMegaBytes=CounterValue // ...and rename the counter value 278 | | distinct Computer // Now weed out the duplicate rows as 279 | , TimeGenerated 280 | , CounterName // ...the perf table will have 281 | , FreeMegaBytes // ...multiple entries during the day 282 | | top 25 by FreeMegaBytes asc // Filter to most critical ones 283 | 284 | 285 | // **Scalar operators** 286 | // Scalar operators allow us to format and transform data, and logical operators for "if then" logic. 287 | 288 | // **Print** retrieves an output that is defined or calculated 289 | print "Hello world" 290 | 291 | // This command is useful for debugging a calculation stepwise 292 | // You can also name the output 293 | // print something="something" 294 | 295 | 296 | // Get the time in UTC. This is the standard in all Azure logs. We can use it for any changes to date and time data. 297 | print now() 298 | 299 | 300 | // Print time one hour ago 301 | print ago(1) // 1h 1m 1d 1s 1ms 1microsend 1tick 302 | 303 | 304 | // Print time in the future 305 | print ago(-1d) // print time tomorrow, -365d gives us a year in the future 306 | 307 | // This lets us retrieve values for a specific timerange when combined with other clauses such as where and other operators. 308 | 309 | // **Sort** lets us sort the columns as desired 310 | Perf 311 | | where TimeGenerated > ago(15m) 312 | | where CounterName == "Avg. Disk sec/Read" 313 | and InstanceName == "C:" 314 | | project Computer 315 | , TimeGenerated 316 | , ObjectName 317 | , CounterName 318 | , InstanceName 319 | , CounterValue 320 | | sort by Computer 321 | , TimeGenerated 322 | 323 | 324 | // Sort by ascending 325 | Perf 326 | | where TimeGenerated > ago(15m) 327 | | where CounterName == "Avg. Disk sec/Read" 328 | and InstanceName == "C:" 329 | | project Computer 330 | , TimeGenerated 331 | , ObjectName 332 | , CounterName 333 | , InstanceName 334 | , CounterValue 335 | | sort by Computer asc 336 | , TimeGenerated asc 337 | 338 | 339 | // You can also mix sorting clauses 340 | Perf 341 | | where TimeGenerated > ago(15m) 342 | | where CounterName == "Avg. Disk sec/Read" 343 | and InstanceName == "C:" 344 | | project Computer 345 | , TimeGenerated 346 | , ObjectName 347 | , CounterName 348 | , InstanceName 349 | , CounterValue 350 | | sort by Computer asc 351 | , TimeGenerated 352 | 353 | 354 | // `order by` is an alias for `sort by` 355 | 356 | // **String operators** // 357 | // String operators let us match a specifc string, take into account case sensitivity and use affixes to match several strings in sequence. 358 | 359 | // Extending our previous queries we can supplement with an additional Equals operator that is case-insensitive by using `=~`. This is useful when a log can contain multiple entries with different casing. 360 | Perf 361 | | where TimeGenerated > ago(15m) 362 | | where CounterName == "Avg. Disk sec/Read" and InstanceName =~ "SDA" 363 | | take 10 364 | 365 | 366 | // The affixes in Kusto offer very specific string matching, including `prefix`, `suffix` for the beginning and ending of a string respectively 367 | Perf 368 | | where TimeGenerated > ago(15m) 369 | | where CounterName == "Avg. Disk sec/Read" and Computer hassuffix "cluster-master" 370 | | take 10 371 | 372 | 373 | // There are also sequential matches and negated matches 374 | Perf 375 | | where TimeGenerated > ago(15m) 376 | | where CounterName == "Avg. Disk sec/Read" and Computer hasprefix "gangams-kind" 377 | | take 10 378 | 379 | 380 | // Negated matches also work, but are slower 381 | Perf 382 | | where TimeGenerated > ago(15m) 383 | | where CounterName == "Avg. Disk sec/Read" and Computer !hasprefix "gangams-kind" 384 | | take 10 385 | 386 | // If we are looking for the presence of a specific term it is much faster to look it up using `has` and `in`, than using `contains`, `startswith` and `endswith`. 387 | Perf 388 | | where TimeGenerated > ago(15m) 389 | | where CounterName == "Avg. Disk sec/Read" and InstanceName has "SDA" 390 | | take 10 391 | 392 | // Microsoft's official example of this is to compare these queries 393 | EventLog | where continent has "North" | count; 394 | EventLog | where continent contains "nor" | count 395 | 396 | //The reason the first query runs faster is because Kusto indexes all columns including those of type string. Words consisting of over 4 characters are treated as **terms**. These are transformed into sequences of alphanumeric characters, and therefore an exact match can be run much faster on these words. 397 | 398 | // See the [String Operators documentation](https://docs.microsoft.com/en-us/azure/kusto/query/datatypes-string-operators) for more information on this feature. It will help you choose different operators to increase performance of your queries. 399 | 400 | // **Extract** 401 | // Extract will match a string based on a regular expression pattern, and retrieves only the part that matches. 402 | Perf 403 | | where ObjectName == "LogicalDisk" 404 | and InstanceName matches regex "[A-Z]:" 405 | | project Computer 406 | , CounterName 407 | , extract("[A-Z]:", 0, InstanceName) 408 | 409 | 410 | // Extract only the disk name without the colon 411 | Perf 412 | | where ObjectName == "LogicalDisk" 413 | and InstanceName matches regex "[A-Z]:" 414 | | project Computer 415 | , CounterName 416 | , extract("([A-Z]):", 1, InstanceName) 417 | 418 | 419 | // **Parse** 420 | // Parse takes a text string and extracts part of it into a column name using markers 421 | // `Parse` runs until it has parsed the entire dataset or reached the final match we've specified. 422 | 423 | // Parse is very useful when you have large blobs of text you want to turn into standard components 424 | Event 425 | | where RenderedDescription startswith "Event code:" 426 | | parse RenderedDescription with "Event code: " myEventCode 427 | " Event message: " myEventMessage 428 | " Event time: " myEventTime 429 | " Event time (UTC): " myEventTimeUTC 430 | " Event ID: " myEventID 431 | " Event sequence: " myEventSequence 432 | " Event occurrence: " * 433 | | project myEventCode, myEventMessage, myEventTime, myEventTimeUTC, myEventID, myEventSequence 434 | 435 | 436 | // **datetime arithmetic** 437 | // Convert a string into a datetime for our query - for year to date, using `datetime` 438 | Perf 439 | | where CounterName == "Avg. Disk sec/Read" 440 | | where CounterValue > 0 441 | | take 100 442 | | extend HowLongAgo=( now() - TimeGenerated ) 443 | , TimeSinceStartofYear=( TimeGenerated - datetime(2018-01-01) ) 444 | | project Computer 445 | , CounterName 446 | , CounterValue 447 | , TimeGenerated 448 | , HowLongAgo 449 | , TimeSinceStartofYear 450 | 451 | 452 | 453 | // Converting a datetime, f.ex into hours can be done with simple arithmetic of division 454 | Perf 455 | | where CounterName == "Avg. Disk sec/Read" 456 | | where CounterValue > 0 457 | | take 100 458 | | extend HowLongAgo=( now() - TimeGenerated ) 459 | , TimeSinceStartOfYear=( TimeGenerated - datetime(2018-01-01) ) 460 | | extend TimeSinceStartOfYearInHours=( TimeSinceStartOfYear / 1h) 461 | | project Computer 462 | , CounterName 463 | , CounterValue 464 | , TimeGenerated 465 | , HowLongAgo 466 | , TimeSinceStartOfYear 467 | , TimeSinceStartOfYearInHours 468 | 469 | 470 | 471 | // Simple datetime calculations over columns 472 | Usage 473 | | extend Duration=( EndTime - StartTime ) 474 | | project Computer 475 | , StartTime 476 | , EndTime 477 | , Duration 478 | 479 | 480 | // Combining summarize with datetime functions by a specific timeperiod 481 | Event 482 | | where TimeGenerated >= ago(7d) 483 | | extend DayGenerated = startofday(TimeGenerated) 484 | | project Source 485 | , DayGenerated 486 | | summarize EventCount=count() 487 | by DayGenerated 488 | , Source 489 | 490 | // Retrieves number of events per source and for the last 7 days. 491 | Event 492 | | where TimeGenerated >= ago(365d) 493 | | extend MonthGenerated = startofmonth(TimeGenerated) 494 | | project Source 495 | , MonthGenerated 496 | | summarize EventCount=count() 497 | by MonthGenerated 498 | , Source 499 | | sort by MonthGenerated desc 500 | , Source asc 501 | 502 | //Retrieves number of events per source by month for the last 365 days. 503 | 504 | // You can also use `startofweek` and `startofyear` for similar operations. 505 | 506 | //There are also corresponding end of functions, f.ex `endofday` `endofweek`, `endofmonth` and `endofyear` 507 | Event 508 | | where TimeGenerated >= ago(7d) 509 | | extend DayGenerated = endofday(TimeGenerated) 510 | | project Source 511 | , DayGenerated 512 | | summarize EventCount=count() 513 | by DayGenerated 514 | , Source 515 | | sort by DayGenerated desc 516 | , Source asc 517 | 518 | 519 | // **Between commands** 520 | // Allows us to specify a range of values, be it numeric or datetime, to retrieve. 521 | Perf 522 | | where CounterName == "% Free Space" 523 | | where CounterValue between( 70.0 .. 100.0 ) 524 | 525 | 526 | // Likewise for dates 527 | Perf 528 | | where CounterName == "% Free Space" 529 | | where TimeGenerated between( datetime(2019-04-01) .. datetime(2019-04-03) ) 530 | | take 10 531 | 532 | 533 | // Gathering data for start of and end of specific dates 534 | Perf 535 | | where CounterName == "% Free Space" 536 | | where TimeGenerated between( startofday(datetime(2019-04-01)) .. endofday(datetime(2019-04-03)) ) 537 | | take 10 538 | 539 | 540 | // There is also a "not between" operator `!between`, which lets us fetch values not within a range - only those outside it. 541 | Perf 542 | | where CounterName == "% Free Space" 543 | | where CounterValue !between ( 0.0 .. 69.9999 ) 544 | | take 10 545 | 546 | 547 | // **Todynamic** 548 | // Takes json stored in a string and lets you retrieve its individual values 549 | // Convert json to a variable using `todynamic`, then step into the json array and project the values 550 | // Use the key as column names 551 | SecurityAlert 552 | | where TimeGenerated > ago(365d) 553 | | extend Extprops=todynamic(ExtendedProperties) 554 | | project AlertName 555 | , TimeGenerated 556 | , Extprops["Alert Start Time (UTC)"] 557 | , Extprops["Source"] 558 | , Extprops["Non-Existent Users"] 559 | , Extprops["Existing Users"] 560 | , Extprops["Failed Attempts"] 561 | , Extprops["Successful Logins"] 562 | , Extprops["Successful User Logons"] 563 | , Extprops["Account Logon Ids"] 564 | , Extprops["Failed User Logons"] 565 | , Extprops["End Time UTC"] 566 | , Extprops["ActionTaken"] 567 | , Extprops["resourceType"] 568 | , Extprops["ServiceId"] 569 | , Extprops["ReportingSystem"] 570 | , Extprops["OccuringDatacenter"] 571 | 572 | 573 | // You can use column-renaming to structure the output better 574 | SecurityAlert 575 | | where TimeGenerated > ago(365d) 576 | | extend Extprops=todynamic(ExtendedProperties) 577 | | project AlertName 578 | , TimeGenerated 579 | , AlertStartTime = Extprops["Alert Start Time (UTC)"] 580 | , Source = Extprops["Source"] 581 | , NonExistentUsers = Extprops["Non-Existent Users"] 582 | , ExistingUsers = Extprops["Existing Users"] 583 | , FailedAttempts = Extprops["Failed Attempts"] 584 | , SuccessfulLogins = Extprops["Successful Logins"] 585 | , SuccessfulUserLogons = Extprops["Successful User Logons"] 586 | , AccountLogonIds = Extprops["Account Logon Ids"] 587 | , FailedUserLogons= Extprops["Failed User Logons"] 588 | , EndTimeUtc = Extprops["End Time UTC"] 589 | , ActionTaken = Extprops["ActionTaken"] 590 | , ResourceType = Extprops["resourceType"] 591 | , ServiceId = Extprops["ServiceId"] 592 | , ReportingSystem = Extprops["ReportingSystem"] 593 | , OccuringDataCenter = Extprops["OccuringDatacenter"] 594 | 595 | 596 | 597 | // If the JSON keys do not have spaces you can also use property notation 598 | SecurityAlert 599 | | where TimeGenerated > ago(365d) 600 | | extend Extprops=todynamic(ExtendedProperties) 601 | | project AlertName 602 | , TimeGenerated 603 | , AlertStartTime = Extprops["Alert Start Time (UTC)"] 604 | , Source = Extprops.Source 605 | , NonExistentUsers = Extprops["Non-Existent Users"] 606 | , ExistingUsers = Extprops["Existing Users"] 607 | , FailedAttempts = Extprops["Failed Attempts"] 608 | , SuccessfulLogins = Extprops["Successful Logins"] 609 | , SuccessfulUserLogons = Extprops["Successful User Logons"] 610 | , AccountLogonIds = Extprops["Account Logon Ids"] 611 | , FailedUserLogons= Extprops["Failed User Logons"] 612 | , EndTimeUtc = Extprops["End Time UTC"] 613 | , ActionTaken = Extprops.ActionTaken 614 | , ResourceType = Extprops.resourceType 615 | , ServiceId = Extprops.ServiceId 616 | , ReportingSystem = Extprops.ReportingSystem 617 | , OccuringDataCenter = Extprops.OccuringDatacenter 618 | 619 | 620 | // Multilevel notation is also supported, f.ex `Extprops.Level1.Level2` 621 | 622 | // **format_datetime** 623 | // Allows you to return specific date formats 624 | Perf 625 | | take 100 626 | | project CounterName 627 | , CounterValue 628 | , TimeGenerated 629 | , format_datetime(TimeGenerated, "y-M-d") 630 | , format_datetime(TimeGenerated, "yyyy-MM-dd") 631 | , format_datetime(TimeGenerated, "MM/dd/yyyy") 632 | , format_datetime(TimeGenerated, "MM/dd/yyyy hh:mm:ss tt") 633 | , format_datetime(TimeGenerated, "MM/dd/yyyy HH:MM:ss") 634 | , format_datetime(TimeGenerated, "MM/dd/yyyy HH:mm:ss.ffff") 635 | 636 | 637 | 638 | 639 | 640 | // **Calculating KPIs** 641 | 642 | // DAU to MAU activity ratio - Daily Active Users to Monthly Active Users 643 | pageViews | union * 644 | | where timestamp > ago(90d) 645 | | evaluate activity_engagement(user_Id, timestamp, 1d, 28d) 646 | | project timestamp, Dau_Mau=activity_ratio*100 647 | | where timestamp > ago(62d) // remove tail with partial data 648 | | render timechart 649 | 650 | // **New Users** 651 | customEvents 652 | | evaluate new_activity_metrics(session_Id, timestamp, startofday(ago(7d)), startofday(now()), 1d) 653 | 654 | 655 | // New Users, Returning and Churned 656 | // https://docs.microsoft.com/en-us/azure/kusto/query/new-activity-metrics-plugin 657 | T | evaluate new_activity_metrics(id, datetime_column, startofday(ago(30d)), startofday(now()), 1d, dim1, dim2, dim3) 658 | 659 | 660 | // Stepping into a JSON object - example: Summarise events 661 | customEvents 662 | | where customDimensions != "" 663 | | extend customDimensions.Properties.event 664 | | where customDimensions contains "Event name" 665 | | take 10 666 | 667 | 668 | // Stepping into a JSON object - example: Counting users with an event 669 | customEvents 670 | | where timestamp > ago(30d) 671 | | where customDimensions != "" 672 | | extend customDimensions.Properties.event 673 | | where customDimensions contains "Event name" 674 | | summarize dcount(user_Id) by bin(timestamp, 1d) 675 | | take 10 676 | 677 | 678 | // Summarize average counter values for a specific machine by a specific metric for a given continuous variable, binning values 679 | Perf 680 | | where TimeGenerated > ago(30d) 681 | | where Computer == "sqlserver-1.contoso.com" 682 | | where CounterName == "Available MBytes" 683 | | summarize avg(CounterValue) by bin(TimeGenerated, 1h) 684 | 685 | 686 | // Search for specific text - WORKS but not much data 687 | customEvents 688 | | search "2019" 689 | 690 | 691 | // **Unique Users, New Users, Returning Users, Lost Users** 692 | // Can be used in Workbooks with Parameter fields - not in Log Analytics 693 | let timeRange = {TimeRange}; 694 | let monthDefinition = {Metric}; 695 | let hlls = union customEvents, pageViews 696 | | where timestamp >= startofmonth(now() - timeRange - 2 * monthDefinition) 697 | | where name in ({Activities}) or '*' in ({Activities}) 698 | {OtherFilters} 699 | | summarize Hlls = hll(user_Id) by bin(timestamp, 1d) 700 | | project DaysToMerge = timestamp, Hlls; 701 | let churnSeriesWithHllsToInclude = materialize(range d from 0d to timeRange step 1d 702 | | extend Day = startofday(now() - d) 703 | | extend R = range(0d, monthDefinition - 1d, 1d) 704 | | mvexpand R 705 | | extend ThisMonth = Day - totimespan(R) 706 | | extend LastMonth = Day - monthDefinition - totimespan(R) 707 | | project Day, ThisMonth, LastMonth); 708 | churnSeriesWithHllsToInclude 709 | | extend DaysToMerge = ThisMonth 710 | | join kind= inner (hlls) on DaysToMerge 711 | | project Day, ThisMonthHlls = Hlls 712 | | union ( 713 | churnSeriesWithHllsToInclude 714 | | extend DaysToMerge = LastMonth 715 | | join kind= inner (hlls) on DaysToMerge 716 | | project Day, LastMonthHlls = Hlls) 717 | | summarize ThisMonth = hll_merge(ThisMonthHlls), LastMonth = hll_merge(LastMonthHlls) by Day 718 | | evaluate dcount_intersect(ThisMonth, LastMonth) 719 | | extend NewUsers = s0 - s1 720 | | extend ChurnedUsers = -1 * (dcount_hll(LastMonth) - s1) // Last Months Users - Returning Users 721 | | project Day, ["Active Users"] = s1 + NewUsers, ["Returning Users"] = s1, ["Lost Users"] = ChurnedUsers, ["New Users"] = NewUsers 722 | 723 | 724 | // Retention for cohorts 725 | 726 | // Finally, calculate the desired metric for each cohort. In this sample we calculate distinct users but you can change 727 | // this to any other metric that would measure the engagement of the cohort members. 728 | | extend 729 | r0 = DistinctUsers(startDate, startDate+7d), 730 | r1 = DistinctUsers(startDate, startDate+14d), 731 | r2 = DistinctUsers(startDate, startDate+21d), 732 | r3 = DistinctUsers(startDate, startDate+28d), 733 | r4 = DistinctUsers(startDate, startDate+35d) 734 | | union (week | where Cohort == startDate + 7d 735 | | extend 736 | r0 = DistinctUsers(startDate+7d, startDate+14d), 737 | r1 = DistinctUsers(startDate+7d, startDate+21d), 738 | r2 = DistinctUsers(startDate+7d, startDate+28d), 739 | r3 = DistinctUsers(startDate+7d, startDate+35d) ) 740 | | union (week | where Cohort == startDate + 14d 741 | | extend 742 | r0 = DistinctUsers(startDate+14d, startDate+21d), 743 | r1 = DistinctUsers(startDate+14d, startDate+28d), 744 | r2 = DistinctUsers(startDate+14d, startDate+35d) ) 745 | | union (week | where Cohort == startDate + 21d 746 | | extend 747 | r0 = DistinctUsers(startDate+21d, startDate+28d), 748 | r1 = DistinctUsers(startDate+21d, startDate+35d) ) 749 | | union (week | where Cohort == startDate + 28d 750 | | extend 751 | r0 = DistinctUsers (startDate+28d, startDate+35d) ) 752 | // Calculate the retention percentage for each cohort by weeks 753 | | project Cohort, r0, r1, r2, r3, r4, 754 | p0 = r0/r0*100, 755 | p1 = todouble(r1)/todouble (r0)*100, 756 | p2 = todouble(r2)/todouble(r0)*100, 757 | p3 = todouble(r3)/todouble(r0)*100, 758 | p4 = todouble(r4)/todouble(r0)*100 759 | | sort by Cohort asc 760 | 761 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kusto-queries 2 | 3 | Example queries for learning the Kusto Query language in [Azure Data Explorer](https://docs.microsoft.com/en-us/azure/data-explorer/data-explorer-overview). Kusto can be used in Azure Monitor Logs, Application Insights, Time Series Insights and Defender Advanced Threat Perception. 4 | 5 | Azure Data Explorer is a Microsoft service for analysing log and telemetry data. You can use it to log events in your mobile applications and for monitoring devices enrolled in a company network. 6 | 7 | You can start learning KQL at https://portal.loganalytics.io/demo 8 | 9 | **Useful learning resources** 10 | 11 | * [Get started with Azure monitor](https://docs.microsoft.com/en-us/azure/azure-monitor/log-query/get-started-portal) 12 | * https://github.com/MicrosoftDocs/azure-docs/blob/master/articles/data-explorer/write-queries.md 13 | * https://4pp1n51ght5.com/2017/08/23/using-azure-log-analytics-to-calculate-user-engagement-metrics/ 14 | * https://4pp1n51ght5.com/2017/02/08/calculating-stickiness-using-appinsights-analytics/ 15 | 16 | **Contents** 17 | 18 | - [Kusto-queries](#kusto-queries) 19 | - [**Searching**](#searching) 20 | - [**Time and timerange**](#time-and-timerange) 21 | - [**Where**](#where) 22 | - [**Take and Limit command**](#take-and-limit-command) 23 | - [**Count operator**](#count-operator) 24 | - [**Summarize**](#summarize) 25 | - [**Extend**](#extend) 26 | - [**Project command**](#project-command) 27 | - [**Distinct**](#distinct) 28 | - [**Scalar operators**](#scalar-operators) 29 | - [**String operators**](#string-operators) 30 | - [**Extract**](#extract) 31 | - [**Parse**](#parse) 32 | - [**datetime arithmetic**](#datetime-arithmetic) 33 | - [**Between commands**](#between-commands) 34 | - [**Todynamic**](#todynamic) 35 | - [**format_datetime**](#format_datetime) 36 | - [**Calculating KPIs**](#calculating-kpis) 37 | 38 | ## **Searching** 39 | 40 | Search across all datasets 41 | 42 | ```kusto 43 | search "event name" | take 10 44 | ``` 45 | 46 | We can use the `take` command to limit our search to 10 search results. This speeds up our querying substantially. Kusto queries can take a long time to execute if the datasets are large. To avoid this, use the take command before running queries on a full dataset. 47 | 48 | The timeout can take anything from 10 seconds up to 30 minutes. You can cancel your query if you don't want to wait, or allow the query to run and open a new query in a new tab if you need it. 49 | 50 | Searching in a single dataset 51 | ```kusto 52 | Perf | search "Contoso" 53 | ``` 54 | Searching in multiple datasets 55 | ```kusto 56 | search in (Perf, Event, Alert) "Contoso" | take 10 57 | ``` 58 | Searching in specific columns for a match 59 | ```kusto 60 | Perf 61 | | search CounterName=="Available MBytes" | take 10 62 | ``` 63 | Searching for a term anywhere in text 64 | ```kusto 65 | Perf 66 | | search "*Bytes*" | take 10 67 | ``` 68 | Searching for columns that begins with 69 | ```kusto 70 | Perf 71 | | search * startswith "Bytes" | take 10 72 | ``` 73 | Searching for columns that ends with 74 | ```kusto 75 | Perf 76 | | search * endswith "Bytes" | take 10 77 | ``` 78 | Search starts with specific term, anything in between and ends with a specific term 79 | ```kusto 80 | Perf 81 | | search "Free*bytes" | take 10 82 | ``` 83 | 84 | **Combining searches** 85 | 86 | Multiple terms 87 | ```kusto 88 | Perf 89 | | search "Free*bytes" and ("C:" or "D:") | take 10 90 | ``` 91 | Using regular expressions 92 | ```kusto 93 | Perf 94 | | search InstanceName matches regex "[A-Z]:" | take 10 95 | ``` 96 | 97 | ## **Time and timerange** 98 | Dynamic timerange 99 | ```kusto 100 | | where timestamp between (datetime(2019-01-01T00:01:24.615Z)..now()) 101 | ``` 102 | 103 | ## **Where** 104 | 105 | ```kusto 106 | Perf 107 | | where TimeGenerated >= ago(1h) 108 | ``` 109 | Note the time range for the query is automatically set in the query when we use time operators in our where clause 110 | 111 | Using the `and` statement 112 | ```kusto 113 | Perf 114 | | where TimeGenerated >= ago(1h) and CounterName == "Bytes Received/sec" | take 10 115 | ``` 116 | 117 | Using the `or` statement 118 | ```kusto 119 | Perf 120 | | where TimeGenerated >= ago(1h) and (CounterName == "Bytes Received/sec" or CounterName == "% Processor Time") | take 10 121 | ``` 122 | 123 | Stacking multiple where clauses 124 | ```kusto 125 | Perf 126 | | where TimeGenerated >= ago(1h) 127 | | where (CounterName == "Bytes Received/sec" 128 | or 129 | CounterName == "% Processor Time") 130 | | where CounterValue > 0 131 | | take 10 132 | ``` 133 | 134 | Other uses for where - simulating search with where 135 | ```kusto 136 | Perf 137 | | where * has "Bytes" | take 10 138 | ``` 139 | 140 | **Positional matching** with where 141 | 142 | Matches words beginning with Bytes 143 | ```kusto 144 | Perf 145 | | where * hasprefix "Bytes" | take 10 146 | ``` 147 | 148 | Matches words ending with Bytes 149 | ```kusto 150 | Perf 151 | | where * hassuffix "Bytes" | take 10 152 | ``` 153 | 154 | Using `contains` 155 | ```kusto 156 | Perf 157 | | where * contains "Bytes" | take 10 158 | ``` 159 | 160 | Using regular expressions with where 161 | ```kusto 162 | Perf 163 | | where InstanceName matches regex "[A-Z]:" | take 10 164 | ``` 165 | 166 | ## **Take and Limit command** 167 | Take retrieves a random number of rows, f.ex 168 | ```kusto 169 | Perf 170 | | take 10 171 | ``` 172 | Takes 10 rows 173 | 174 | Limit is the equivalent 175 | ```kusto 176 | Perf 177 | | limit 10 178 | ``` 179 | 180 | ## **Count operator** 181 | Count rows in a table 182 | ```kusto 183 | Perf 184 | | count 185 | ``` 186 | Counts all rows in the Perf dataset 187 | 188 | Count can be used with filtering to count rows in a selected table 189 | ```kusto 190 | Perf 191 | | where TimeGenerated >= ago(1h) 192 | and CounterName == "Bytes received/sec" 193 | and CounterValue > 0 194 | | count 195 | ``` 196 | 197 | ## **Summarize** 198 | 199 | Aggregation function for summarizing data in a table, can include `by` 200 | ```kusto 201 | Perf 202 | | summarize count() by CounterName 203 | ``` 204 | 205 | Summarizing multiple columns 206 | ```kusto 207 | Perf 208 | | summarize count() by ObjectName, CounterName 209 | ``` 210 | Note it generates a third column called `count_`. The name is automatically generated. 211 | 212 | If we want to supply a more suitable name as a variable, we can name it 213 | ```kusto 214 | Perf 215 | | summarize PerfCount=count() 216 | by ObjectName, CounterName 217 | ``` 218 | 219 | With Summarize, use other aggregation functions 220 | ```kusto 221 | Perf 222 | | where CounterName == "% Free Space" 223 | | summarize NumberOfEntries=count() 224 | , AverageFreeSpace=avg(CounterValue) 225 | by CounterName 226 | ``` 227 | 228 | Using `Bin` to create logical groups 229 | ```kusto 230 | Perf 231 | | summarize NumberOfEntries=count() 232 | by bin(TimeGenerated, 1d) 233 | ``` 234 | 235 | Using other values for binning 236 | ```kusto 237 | Perf 238 | | where CounterName == "% Free Space" 239 | | summarize NumberOfRowsAtThisPercentLevel=count() 240 | by bin(CounterValue, 10) 241 | ``` 242 | 243 | 244 | ## **Extend** 245 | 246 | Extend allows you to create calculated columns to add to your tables 247 | ```kusto 248 | Perf 249 | | where CounterName == "Free Megabytes" 250 | | extend FreeGB = CounterValue / 1000 251 | ``` 252 | 253 | You can also create multiple columns 254 | ```kusto 255 | Perf 256 | | where CounterName == "Free Megabytes" 257 | | extend FreeGB = CounterValue / 1000 258 | , FreeKB = CounterValue * 1000 259 | ``` 260 | 261 | Repeating a column 262 | ```kusto 263 | Perf 264 | | where CounterName == "Free Megabytes" 265 | | extend FreeGB = CounterValue / 1000 266 | , FreeMB = CounterValue 267 | , FreeKB = CounterValue * 1000 268 | ``` 269 | 270 | Create new string values 271 | 272 | We can create a new column with string values 273 | ```kusto 274 | Perf 275 | | where TimeGenerated >= ago(10m) 276 | | extend ObjectCounter = strcat(ObjectName, " - ", CounterName) 277 | ``` 278 | We use `strcat` to concatenate strings 279 | 280 | ## **Project command** 281 | 282 | Project allows us to select which columns we want in our table 283 | ```kusto 284 | Perf 285 | | project ObjectName 286 | , CounterName 287 | , InstanceName 288 | , CounterValue 289 | , TimeGenerated 290 | ``` 291 | 292 | Project and Extend are very useful when creating tables with our specific data. 293 | ```kusto 294 | Perf 295 | | project ObjectName 296 | , CounterName 297 | , InstanceName 298 | , CounterValue 299 | , TimeGenerated 300 | | extend FreeGB = CounterValue / 1000 301 | , FreeMB = CounterValue 302 | , FreeKB = CounterValue * 1000 303 | ``` 304 | 305 | If we want to omit a specific column we must calculate our values first, then project afterwards 306 | ```kusto 307 | Perf 308 | | extend FreeGB = CounterValue / 1000 309 | , FreeMB = CounterValue 310 | , FreeKB = CounterValue * 1000 311 | | project ObjectName 312 | , CounterName 313 | , InstanceName 314 | , TimeGenerated 315 | , FreeGB 316 | , FreeMB 317 | , FreeKB 318 | 319 | ``` 320 | 321 | OR we can just project our data and calculate it on the fly 322 | ```kusto 323 | Perf 324 | | where CounterName == "Free Megabytes" 325 | | project ObjectName 326 | , CounterName 327 | , InstanceName 328 | , TimeGenerated 329 | , FreeGB = CounterValue / 1000 330 | , FreeMB = CounterValue 331 | , FreeKB = CounterValue * 1000 332 | 333 | ``` 334 | 335 | `Project-away` lets you remove selected columns from the output 336 | ```kusto 337 | Perf 338 | | where CounterName == "Free Megabytes" 339 | | project-away TenantId 340 | , SourceSystem 341 | , CounterPath 342 | , MG 343 | ``` 344 | 345 | `Project-rename` lets you rename a specific column 346 | ```kusto 347 | Perf 348 | | where TimeGenerated > ago(1h) 349 | | project-rename myRenamedComputer = Computer 350 | ``` 351 | 352 | ## **Distinct** 353 | 354 | Retrieves a list of deduplicated values 355 | ```kusto 356 | Perf 357 | | distinct ObjectName, CounterName 358 | ``` 359 | 360 | Only show unique error events 361 | ```kusto 362 | Event 363 | | where EventLevelName == "Error" 364 | | distinct Source 365 | ``` 366 | 367 | Using top for the first 20 rows 368 | ```kusto 369 | Perf 370 | | top 20 by TimeGenerated desc 371 | ``` 372 | Sort rows ascending with `asc` or descending with `desc` 373 | 374 | Combining everything so far 375 | ```kusto 376 | Perf 377 | | where CounterName == "Free Megabytes" // Get the free Megabytes 378 | and TimeGenerated >= ago(1h) // ... within the last hour 379 | | project Computer // For each return the Computer Name 380 | , TimeGenerated // ...and when the counter was generated 381 | , CounterName // ...and the Counter Name 382 | , FreeMegaBytes=CounterValue // ...and rename the counter value 383 | | distinct Computer // Now weed out the duplicate rows as 384 | , TimeGenerated 385 | , CounterName // ...the perf table will have 386 | , FreeMegaBytes // ...multiple entries during the day 387 | | top 25 by FreeMegaBytes asc // Filter to most critical ones 388 | ``` 389 | 390 | ## **Scalar operators** 391 | 392 | Scalar operators allow us to format and transform data, and logical operators for "if then" logic. 393 | 394 | **Print** retrieves an output that is defined or calculated 395 | ```kusto 396 | print "Hello world" 397 | ``` 398 | This command is useful for debugging a calculation stepwise 399 | 400 | You can also name the output 401 | ```kusto 402 | print something="something" 403 | ``` 404 | 405 | Get the time in UTC. This is the standard in all Azure logs. We can use it for any changes to date and time data. 406 | ```kusto 407 | print now() 408 | ``` 409 | 410 | Print time one hour ago 411 | ```kusto 412 | print ago(1) // 1h 1m 1d 1s 1ms 1microsend 1tick 413 | ``` 414 | 415 | Print time in the future 416 | ```kusto 417 | print ago(-1d) // print time tomorrow, -365d gives us a year in the future 418 | ``` 419 | 420 | This lets us retrieve values for a specific timerange when combined with other clauses such as where and other operators. 421 | 422 | **Sort** lets us sort the columns as desired 423 | ```kusto 424 | Perf 425 | | where TimeGenerated > ago(15m) 426 | | where CounterName == "Avg. Disk sec/Read" 427 | and InstanceName == "C:" 428 | | project Computer 429 | , TimeGenerated 430 | , ObjectName 431 | , CounterName 432 | , InstanceName 433 | , CounterValue 434 | | sort by Computer 435 | , TimeGenerated 436 | ``` 437 | 438 | Sort by ascending 439 | ```kusto 440 | Perf 441 | | where TimeGenerated > ago(15m) 442 | | where CounterName == "Avg. Disk sec/Read" 443 | and InstanceName == "C:" 444 | | project Computer 445 | , TimeGenerated 446 | , ObjectName 447 | , CounterName 448 | , InstanceName 449 | , CounterValue 450 | | sort by Computer asc 451 | , TimeGenerated asc 452 | ``` 453 | 454 | You can also mix sorting clauses 455 | ```kusto 456 | Perf 457 | | where TimeGenerated > ago(15m) 458 | | where CounterName == "Avg. Disk sec/Read" 459 | and InstanceName == "C:" 460 | | project Computer 461 | , TimeGenerated 462 | , ObjectName 463 | , CounterName 464 | , InstanceName 465 | , CounterValue 466 | | sort by Computer asc 467 | , TimeGenerated 468 | ``` 469 | 470 | `order by` is an alias for `sort by` 471 | 472 | ## **String operators** 473 | 474 | String operators let us match a specifc string, take into account case sensitivity and use affixes to match several strings in sequence. 475 | 476 | Extending our previous queries we can supplement with an additional Equals operator that is case-insensitive by using `=~`. This is useful when a log can contain multiple entries with different casing. 477 | ```kusto 478 | Perf 479 | | where TimeGenerated > ago(15m) 480 | | where CounterName == "Avg. Disk sec/Read" and InstanceName =~ "SDA" 481 | | take 10 482 | ``` 483 | 484 | The affixes in Kusto offer very specific string matching, including `prefix`, `suffix` for the beginning and ending of a string respectively 485 | ```kusto 486 | Perf 487 | | where TimeGenerated > ago(15m) 488 | | where CounterName == "Avg. Disk sec/Read" and Computer hassuffix "cluster-master" 489 | | take 10 490 | ``` 491 | 492 | There are also sequential matches and negated matches 493 | ```kusto 494 | Perf 495 | | where TimeGenerated > ago(15m) 496 | | where CounterName == "Avg. Disk sec/Read" and Computer hasprefix "gangams-kind" 497 | | take 10 498 | ``` 499 | 500 | Negated matches also work, but are slower 501 | ```kusto 502 | Perf 503 | | where TimeGenerated > ago(15m) 504 | | where CounterName == "Avg. Disk sec/Read" and Computer !hasprefix "gangams-kind" 505 | | take 10 506 | ``` 507 | 508 | If we are looking for the presence of a specific term it is much faster to look it up using `has` and `in`, than using `contains`, `startswith` and `endswith`. 509 | ```kusto 510 | Perf 511 | | where TimeGenerated > ago(15m) 512 | | where CounterName == "Avg. Disk sec/Read" and InstanceName has "SDA" 513 | | take 10 514 | ``` 515 | 516 | Microsoft's official example of this is to compare these queries 517 | ```kusto 518 | EventLog | where continent has "North" | count; 519 | EventLog | where continent contains "nor" | count 520 | ``` 521 | 522 | The reason the first query runs faster is because Kusto indexes all columns including those of type string. Words consisting of over 4 characters are treated as **terms**. These are transformed into sequences of alphanumeric characters, and therefore an exact match can be run much faster on these words. 523 | 524 | See the [String Operators documentation](https://docs.microsoft.com/en-us/azure/kusto/query/datatypes-string-operators) for more information on this feature. It will help you choose different operators to increase performance of your queries. 525 | 526 | ## **Extract** 527 | Extract will match a string based on a regular expression pattern, and retrieves only the part that matches. 528 | 529 | ```kusto 530 | Perf 531 | | where ObjectName == "LogicalDisk" 532 | and InstanceName matches regex "[A-Z]:" 533 | | project Computer 534 | , CounterName 535 | , extract("[A-Z]:", 0, InstanceName) 536 | ``` 537 | 538 | Extract only the disk name without the colon 539 | ```kusto 540 | Perf 541 | | where ObjectName == "LogicalDisk" 542 | and InstanceName matches regex "[A-Z]:" 543 | | project Computer 544 | , CounterName 545 | , extract("([A-Z]):", 1, InstanceName) 546 | ``` 547 | 548 | ## **Parse** 549 | Parse takes a text string and extracts part of it into a column name using markers 550 | 551 | `Parse` runs until it has parsed the entire dataset or reached the final match we've specified. 552 | 553 | Parse is very useful when you have large blobs of text you want to turn into standard components 554 | 555 | ```kusto 556 | Event 557 | | where RenderedDescription startswith "Event code:" 558 | | parse RenderedDescription with "Event code: " myEventCode 559 | " Event message: " myEventMessage 560 | " Event time: " myEventTime 561 | " Event time (UTC): " myEventTimeUTC 562 | " Event ID: " myEventID 563 | " Event sequence: " myEventSequence 564 | " Event occurrence: " * 565 | | project myEventCode, myEventMessage, myEventTime, myEventTimeUTC, myEventID, myEventSequence 566 | ``` 567 | 568 | ## **datetime arithmetic** 569 | 570 | Convert a string into a datetime for our query - for year to date, using `datetime` 571 | ```kusto 572 | Perf 573 | | where CounterName == "Avg. Disk sec/Read" 574 | | where CounterValue > 0 575 | | take 100 576 | | extend HowLongAgo=( now() - TimeGenerated ) 577 | , TimeSinceStartofYear=( TimeGenerated - datetime(2018-01-01) ) 578 | | project Computer 579 | , CounterName 580 | , CounterValue 581 | , TimeGenerated 582 | , HowLongAgo 583 | , TimeSinceStartofYear 584 | 585 | ``` 586 | 587 | Converting a datetime, f.ex into hours can be done with simple arithmetic of division 588 | ```kusto 589 | Perf 590 | | where CounterName == "Avg. Disk sec/Read" 591 | | where CounterValue > 0 592 | | take 100 593 | | extend HowLongAgo=( now() - TimeGenerated ) 594 | , TimeSinceStartOfYear=( TimeGenerated - datetime(2018-01-01) ) 595 | | extend TimeSinceStartOfYearInHours=( TimeSinceStartOfYear / 1h) 596 | | project Computer 597 | , CounterName 598 | , CounterValue 599 | , TimeGenerated 600 | , HowLongAgo 601 | , TimeSinceStartOfYear 602 | , TimeSinceStartOfYearInHours 603 | 604 | ``` 605 | 606 | Simple datetime calculations over columns 607 | ```kusto 608 | Usage 609 | | extend Duration=( EndTime - StartTime ) 610 | | project Computer 611 | , StartTime 612 | , EndTime 613 | , Duration 614 | ``` 615 | 616 | Combining summarize with datetime functions by a specific timeperiod 617 | ```kusto 618 | Event 619 | | where TimeGenerated >= ago(7d) 620 | | extend DayGenerated = startofday(TimeGenerated) 621 | | project Source 622 | , DayGenerated 623 | | summarize EventCount=count() 624 | by DayGenerated 625 | , Source 626 | ``` 627 | Retrieves number of events per source and for the last 7 days. 628 | 629 | ```kusto 630 | Event 631 | | where TimeGenerated >= ago(365d) 632 | | extend MonthGenerated = startofmonth(TimeGenerated) 633 | | project Source 634 | , MonthGenerated 635 | | summarize EventCount=count() 636 | by MonthGenerated 637 | , Source 638 | | sort by MonthGenerated desc 639 | , Source asc 640 | ``` 641 | Retrieves number of events per source by month for the last 365 days. 642 | 643 | You can also use `startofweek` and `startofyear` for similar operations. 644 | 645 | There are also corresponding end of functions, f.ex `endofday` `endofweek`, `endofmonth` and `endofyear` 646 | ```kusto 647 | Event 648 | | where TimeGenerated >= ago(7d) 649 | | extend DayGenerated = endofday(TimeGenerated) 650 | | project Source 651 | , DayGenerated 652 | | summarize EventCount=count() 653 | by DayGenerated 654 | , Source 655 | | sort by DayGenerated desc 656 | , Source asc 657 | ``` 658 | 659 | ## **Between commands** 660 | Allows us to specify a range of values, be it numeric or datetime, to retrieve. 661 | 662 | ```kusto 663 | Perf 664 | | where CounterName == "% Free Space" 665 | | where CounterValue between( 70.0 .. 100.0 ) 666 | ``` 667 | 668 | Likewise for dates 669 | ```kusto 670 | Perf 671 | | where CounterName == "% Free Space" 672 | | where TimeGenerated between( datetime(2019-04-01) .. datetime(2019-04-03) ) 673 | | take 10 674 | ``` 675 | 676 | Gathering data for start of and end of specific dates 677 | ```kusto 678 | Perf 679 | | where CounterName == "% Free Space" 680 | | where TimeGenerated between( startofday(datetime(2019-04-01)) .. endofday(datetime(2019-04-03)) ) 681 | | take 10 682 | ``` 683 | 684 | There is also a "not between" operator `!between`, which lets us fetch values not within a range - only those outside it. 685 | ```kusto 686 | Perf 687 | | where CounterName == "% Free Space" 688 | | where CounterValue !between ( 0.0 .. 69.9999 ) 689 | | take 10 690 | ``` 691 | 692 | ## **Todynamic** 693 | 694 | Takes json stored in a string and lets you retrieve its individual values 695 | 696 | Convert json to a variable using `todynamic`, then step into the json array and project the values 697 | 698 | Use the key as column names 699 | 700 | ```kusto 701 | SecurityAlert 702 | | where TimeGenerated > ago(365d) 703 | | extend Extprops=todynamic(ExtendedProperties) 704 | | project AlertName 705 | , TimeGenerated 706 | , Extprops["Alert Start Time (UTC)"] 707 | , Extprops["Source"] 708 | , Extprops["Non-Existent Users"] 709 | , Extprops["Existing Users"] 710 | , Extprops["Failed Attempts"] 711 | , Extprops["Successful Logins"] 712 | , Extprops["Successful User Logons"] 713 | , Extprops["Account Logon Ids"] 714 | , Extprops["Failed User Logons"] 715 | , Extprops["End Time UTC"] 716 | , Extprops["ActionTaken"] 717 | , Extprops["resourceType"] 718 | , Extprops["ServiceId"] 719 | , Extprops["ReportingSystem"] 720 | , Extprops["OccuringDatacenter"] 721 | 722 | ``` 723 | 724 | You can use column-renaming to structure the output better 725 | 726 | ```kusto 727 | SecurityAlert 728 | | where TimeGenerated > ago(365d) 729 | | extend Extprops=todynamic(ExtendedProperties) 730 | | project AlertName 731 | , TimeGenerated 732 | , AlertStartTime = Extprops["Alert Start Time (UTC)"] 733 | , Source = Extprops["Source"] 734 | , NonExistentUsers = Extprops["Non-Existent Users"] 735 | , ExistingUsers = Extprops["Existing Users"] 736 | , FailedAttempts = Extprops["Failed Attempts"] 737 | , SuccessfulLogins = Extprops["Successful Logins"] 738 | , SuccessfulUserLogons = Extprops["Successful User Logons"] 739 | , AccountLogonIds = Extprops["Account Logon Ids"] 740 | , FailedUserLogons= Extprops["Failed User Logons"] 741 | , EndTimeUtc = Extprops["End Time UTC"] 742 | , ActionTaken = Extprops["ActionTaken"] 743 | , ResourceType = Extprops["resourceType"] 744 | , ServiceId = Extprops["ServiceId"] 745 | , ReportingSystem = Extprops["ReportingSystem"] 746 | , OccuringDataCenter = Extprops["OccuringDatacenter"] 747 | 748 | ``` 749 | 750 | If the JSON keys do not have spaces you can also use property notation 751 | 752 | ```kusto 753 | SecurityAlert 754 | | where TimeGenerated > ago(365d) 755 | | extend Extprops=todynamic(ExtendedProperties) 756 | | project AlertName 757 | , TimeGenerated 758 | , AlertStartTime = Extprops["Alert Start Time (UTC)"] 759 | , Source = Extprops.Source 760 | , NonExistentUsers = Extprops["Non-Existent Users"] 761 | , ExistingUsers = Extprops["Existing Users"] 762 | , FailedAttempts = Extprops["Failed Attempts"] 763 | , SuccessfulLogins = Extprops["Successful Logins"] 764 | , SuccessfulUserLogons = Extprops["Successful User Logons"] 765 | , AccountLogonIds = Extprops["Account Logon Ids"] 766 | , FailedUserLogons= Extprops["Failed User Logons"] 767 | , EndTimeUtc = Extprops["End Time UTC"] 768 | , ActionTaken = Extprops.ActionTaken 769 | , ResourceType = Extprops.resourceType 770 | , ServiceId = Extprops.ServiceId 771 | , ReportingSystem = Extprops.ReportingSystem 772 | , OccuringDataCenter = Extprops.OccuringDatacenter 773 | ``` 774 | 775 | Multilevel notation is also supported, f.ex `Extprops.Level1.Level2` 776 | 777 | ## **format_datetime** 778 | Allows you to return specific date formats 779 | 780 | ```kusto 781 | Perf 782 | | take 100 783 | | project CounterName 784 | , CounterValue 785 | , TimeGenerated 786 | , format_datetime(TimeGenerated, "y-M-d") 787 | , format_datetime(TimeGenerated, "yyyy-MM-dd") 788 | , format_datetime(TimeGenerated, "MM/dd/yyyy") 789 | , format_datetime(TimeGenerated, "MM/dd/yyyy hh:mm:ss tt") 790 | , format_datetime(TimeGenerated, "MM/dd/yyyy HH:MM:ss") 791 | , format_datetime(TimeGenerated, "MM/dd/yyyy HH:mm:ss.ffff") 792 | ``` 793 | 794 | 795 | 796 | 797 | ## **Calculating KPIs** 798 | 799 | DAU to MAU activity ratio - Daily Active Users to Monthly Active Users 800 | ```kusto 801 | pageViews | union * 802 | | where timestamp > ago(90d) 803 | | evaluate activity_engagement(user_Id, timestamp, 1d, 28d) 804 | | project timestamp, Dau_Mau=activity_ratio*100 805 | | where timestamp > ago(62d) // remove tail with partial data 806 | | render timechart 807 | ``` 808 | 809 | New Users 810 | ```kusto 811 | customEvents 812 | | evaluate new_activity_metrics(session_Id, timestamp, startofday(ago(7d)), startofday(now()), 1d) 813 | ``` 814 | 815 | New Users, Returning and Churned 816 | https://docs.microsoft.com/en-us/azure/kusto/query/new-activity-metrics-plugin 817 | ```kusto 818 | T | evaluate new_activity_metrics(id, datetime_column, startofday(ago(30d)), startofday(now()), 1d, dim1, dim2, dim3) 819 | ``` 820 | 821 | Stepping into a JSON object - example: Summarise events 822 | ```kusto 823 | customEvents 824 | | where customDimensions != "" 825 | | extend customDimensions.Properties.event 826 | | where customDimensions contains "Event name" 827 | | take 10 828 | ``` 829 | 830 | Stepping into a JSON object - example: Counting users with an event 831 | ```kusto 832 | customEvents 833 | | where timestamp > ago(30d) 834 | | where customDimensions != "" 835 | | extend customDimensions.Properties.event 836 | | where customDimensions contains "Event name" 837 | | summarize dcount(user_Id) by bin(timestamp, 1d) 838 | | take 10 839 | ``` 840 | 841 | Summarize average counter values for a specific machine by a specific metric for a given continuous variable, binning values 842 | 843 | ```kusto 844 | Perf 845 | | where TimeGenerated > ago(30d) 846 | | where Computer == "sqlserver-1.contoso.com" 847 | | where CounterName == "Available MBytes" 848 | | summarize avg(CounterValue) by bin(TimeGenerated, 1h) 849 | ``` 850 | 851 | Search for specific text - WORKS but not much data 852 | ```kusto 853 | customEvents 854 | | search "2019" 855 | ``` 856 | 857 | Unique Users, New Users, Returning Users, Lost Users 858 | Can be used in Workbooks with Parameter fields - not in Log Analytics 859 | ```kusto 860 | let timeRange = {TimeRange}; 861 | let monthDefinition = {Metric}; 862 | let hlls = union customEvents, pageViews 863 | | where timestamp >= startofmonth(now() - timeRange - 2 * monthDefinition) 864 | | where name in ({Activities}) or '*' in ({Activities}) 865 | {OtherFilters} 866 | | summarize Hlls = hll(user_Id) by bin(timestamp, 1d) 867 | | project DaysToMerge = timestamp, Hlls; 868 | let churnSeriesWithHllsToInclude = materialize(range d from 0d to timeRange step 1d 869 | | extend Day = startofday(now() - d) 870 | | extend R = range(0d, monthDefinition - 1d, 1d) 871 | | mvexpand R 872 | | extend ThisMonth = Day - totimespan(R) 873 | | extend LastMonth = Day - monthDefinition - totimespan(R) 874 | | project Day, ThisMonth, LastMonth); 875 | churnSeriesWithHllsToInclude 876 | | extend DaysToMerge = ThisMonth 877 | | join kind= inner (hlls) on DaysToMerge 878 | | project Day, ThisMonthHlls = Hlls 879 | | union ( 880 | churnSeriesWithHllsToInclude 881 | | extend DaysToMerge = LastMonth 882 | | join kind= inner (hlls) on DaysToMerge 883 | | project Day, LastMonthHlls = Hlls) 884 | | summarize ThisMonth = hll_merge(ThisMonthHlls), LastMonth = hll_merge(LastMonthHlls) by Day 885 | | evaluate dcount_intersect(ThisMonth, LastMonth) 886 | | extend NewUsers = s0 - s1 887 | | extend ChurnedUsers = -1 * (dcount_hll(LastMonth) - s1) // Last Months Users - Returning Users 888 | | project Day, ["Active Users"] = s1 + NewUsers, ["Returning Users"] = s1, ["Lost Users"] = ChurnedUsers, ["New Users"] = NewUsers 889 | ``` 890 | 891 | Retention for cohorts 892 | ```kusto 893 | // Finally, calculate the desired metric for each cohort. In this sample we calculate distinct users but you can change 894 | // this to any other metric that would measure the engagement of the cohort members. 895 | | extend 896 | r0 = DistinctUsers(startDate, startDate+7d), 897 | r1 = DistinctUsers(startDate, startDate+14d), 898 | r2 = DistinctUsers(startDate, startDate+21d), 899 | r3 = DistinctUsers(startDate, startDate+28d), 900 | r4 = DistinctUsers(startDate, startDate+35d) 901 | | union (week | where Cohort == startDate + 7d 902 | | extend 903 | r0 = DistinctUsers(startDate+7d, startDate+14d), 904 | r1 = DistinctUsers(startDate+7d, startDate+21d), 905 | r2 = DistinctUsers(startDate+7d, startDate+28d), 906 | r3 = DistinctUsers(startDate+7d, startDate+35d) ) 907 | | union (week | where Cohort == startDate + 14d 908 | | extend 909 | r0 = DistinctUsers(startDate+14d, startDate+21d), 910 | r1 = DistinctUsers(startDate+14d, startDate+28d), 911 | r2 = DistinctUsers(startDate+14d, startDate+35d) ) 912 | | union (week | where Cohort == startDate + 21d 913 | | extend 914 | r0 = DistinctUsers(startDate+21d, startDate+28d), 915 | r1 = DistinctUsers(startDate+21d, startDate+35d) ) 916 | | union (week | where Cohort == startDate + 28d 917 | | extend 918 | r0 = DistinctUsers (startDate+28d, startDate+35d) ) 919 | // Calculate the retention percentage for each cohort by weeks 920 | | project Cohort, r0, r1, r2, r3, r4, 921 | p0 = r0/r0*100, 922 | p1 = todouble(r1)/todouble (r0)*100, 923 | p2 = todouble(r2)/todouble(r0)*100, 924 | p3 = todouble(r3)/todouble(r0)*100, 925 | p4 = todouble(r4)/todouble(r0)*100 926 | | sort by Cohort asc 927 | ``` 928 | --------------------------------------------------------------------------------