├── .gitignore
├── KQLAnalyzer.Tests
    ├── AnalyzeQueryTests.cs
    ├── KQLAnalyzer.Tests.csproj
    └── test_data
    │   ├── custom_table.json
    │   ├── fileprofile.json
    │   ├── scalar_function.json
    │   ├── sentinel_no_fileprofile.json
    │   ├── simple_query.json
    │   ├── simple_query_using_default_tables.json
    │   ├── tabular_function.json
    │   ├── tabular_function_required_args.json
    │   └── watchlist.json
├── KQLValidator.sln
├── README.md
├── additional_columns.json
├── environments.json
├── get_schemas.py
├── src
    ├── EnvironmentDefinition.cs
    ├── KQLAnalyzer.csproj
    ├── KQLAnalyzerRESTService.cs
    ├── KustoAnalyzer.cs
    ├── Models
    │   ├── AnalyzeRequest.cs
    │   ├── AnalyzeResults.cs
    │   ├── ArgumentDetails.cs
    │   ├── KQLEnvironments.cs
    │   ├── LocalData.cs
    │   ├── ScalarFunctionDetails.cs
    │   ├── TableDetails.cs
    │   ├── TabularFunctionDetails.cs
    │   └── WatchlistDetails.cs
    └── Program.cs
└── update_schemas.sh


/.gitignore:
--------------------------------------------------------------------------------
1 | bin
2 | obj
3 | 


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/AnalyzeQueryTests.cs:
--------------------------------------------------------------------------------
  1 | using Xunit;
  2 | using KQLAnalyzer;
  3 | using System.Text.Json;
  4 | 
  5 | namespace KQLAnalyzerTests
  6 | {
  7 |     public class KQLAnalyzerTests
  8 |     {
  9 |         public static KQLEnvironments kqlEnvironments = JsonSerializer.Deserialize<KQLEnvironments>(
 10 |             File.ReadAllText("environments.json")
 11 |         )!;
 12 | 
 13 |         public static AnalyzeResults AnalyzeFromJson(string inputFile)
 14 |         {
 15 |             var analyzeRequest = JsonSerializer.Deserialize<AnalyzeRequest>(
 16 |                 File.ReadAllText(inputFile)
 17 |             );
 18 | 
 19 |             var environmentName = analyzeRequest!.Environment;
 20 |             var globals = kqlEnvironments[environmentName].ToGlobalState();
 21 | 
 22 |             var results = KustoAnalyzer.AnalyzeQuery(
 23 |                 analyzeRequest.Query,
 24 |                 globals,
 25 |                 analyzeRequest.LocalData
 26 |             );
 27 |             return results;
 28 |         }
 29 | 
 30 |         private static void WriteResults(AnalyzeResults results)
 31 |         {
 32 |             Console.WriteLine(
 33 |                 JsonSerializer.Serialize(
 34 |                     results,
 35 |                     new JsonSerializerOptions { WriteIndented = true }
 36 |                 )
 37 |             );
 38 |         }
 39 | 
 40 |         [Fact]
 41 |         public void SimpleQuery()
 42 |         {
 43 |             var results = AnalyzeFromJson("test_data/simple_query.json");
 44 |             Assert.Empty(results.ParsingErrors);
 45 |             Assert.Equal(results.OutputColumns, new Dictionary<string, string> { { "a", "bool" } });
 46 |         }
 47 | 
 48 |         [Fact]
 49 |         public void SimpleQueryDefaultTables()
 50 |         {
 51 |             var results = AnalyzeFromJson("test_data/simple_query_using_default_tables.json");
 52 |             Assert.Empty(results.ParsingErrors);
 53 |             Assert.Equal(
 54 |                 results.OutputColumns,
 55 |                 new Dictionary<string, string> { { "a", "string" } }
 56 |             );
 57 |         }
 58 | 
 59 | 
 60 |         [Fact]
 61 |         public void SimpleQueryCustomTables()
 62 |         {
 63 |             var results = AnalyzeFromJson("test_data/custom_table.json");
 64 |             Assert.Empty(results.ParsingErrors);
 65 |             Assert.Equal(
 66 |                 results.OutputColumns,
 67 |                 new Dictionary<string, string> { { "Value", "string" } }
 68 |             );
 69 |         }
 70 | 
 71 |         [Fact]
 72 |         public void SentinelNoFileProfile()
 73 |         {
 74 |             var results = AnalyzeFromJson("test_data/sentinel_no_fileprofile.json");
 75 |             Assert.Contains(results.ParsingErrors, (
 76 |                     e => e.Code == "KS211" && e.Message.Contains("FileProfile")
 77 |                 )
 78 |             );
 79 |         }
 80 | 
 81 |         [Fact]
 82 |         public void FileProfile()
 83 |         {
 84 |             var results = AnalyzeFromJson("test_data/fileprofile.json");
 85 |             Assert.Empty(results.ParsingErrors);
 86 |             Assert.Contains(results.OutputColumns,(e => e.Key == "Issuer")); // Add FileProfile columns
 87 |             // SHA1 exists in both input and in FileProfile so there should also be a SHA11
 88 |             Assert.Contains(results.OutputColumns,(e => e.Key == "SHA1"));
 89 |             Assert.Contains(results.OutputColumns,(e => e.Key == "SHA11"));
 90 |             Assert.Contains(results.OutputColumns,(e => e.Key == "Foo")); // Original input column is preserved
 91 |         }
 92 | 
 93 |         [Fact]
 94 |         public void TabularFunction()
 95 |         {
 96 |             var results = AnalyzeFromJson("test_data/tabular_function.json");
 97 |             Assert.Empty(results.ParsingErrors);
 98 |             Assert.Equal(
 99 |                 results.OutputColumns,
100 |                 new Dictionary<string, string> { { "output_foo", "string" } }
101 |             );
102 |             Assert.Equal(results.ReferencedFunctions, new List<string> { "MyFunction" });
103 |         }
104 | 
105 |         [Fact]
106 |         public void TabularFunctionRequiredArgs()
107 |         {
108 |             var results = AnalyzeFromJson("test_data/tabular_function_required_args.json");
109 |             Assert.Contains(results.ParsingErrors,(e => e.Code == "KS119")); // Expect KS119 error The function 'MyFunction' expects 1 argument.
110 |         }
111 | 
112 |         [Fact]
113 |         public void ScalarFunction()
114 |         {
115 |             var results = AnalyzeFromJson("test_data/scalar_function.json");
116 |             Assert.Empty(results.ParsingErrors);
117 |             Assert.Equal(results.OutputColumns, new Dictionary<string, string> { { "a", "bool" } });
118 |             Assert.Equal(results.ReferencedFunctions, new List<string> { "MyScalar" });
119 |         }
120 | 
121 |         [Fact]
122 |         public void Watchlist()
123 |         {
124 |             var results = AnalyzeFromJson("test_data/watchlist.json");
125 |             Assert.Empty(results.ParsingErrors);
126 |             Assert.Equal(
127 |                 results.OutputColumns,
128 |                 new Dictionary<string, string>
129 |                 {
130 |                     { "_DTItemId", "string" },
131 |                     { "LastUpdatedTimeUTC", "datetime" },
132 |                     { "SearchKey", "string" },
133 |                     { "WatchlistItem", "dynamic" },
134 |                     { "foo", "string" },
135 |                 }
136 |             );
137 |             Assert.Equal(results.ReferencedFunctions, new List<string> { "_GetWatchlist" });
138 |         }
139 |     }
140 | }
141 | 


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/KQLAnalyzer.Tests.csproj:
--------------------------------------------------------------------------------
 1 | <Project Sdk="Microsoft.NET.Sdk">
 2 | 
 3 |   <PropertyGroup>
 4 |     <TargetFramework>net8.0</TargetFramework>
 5 |     <ImplicitUsings>enable</ImplicitUsings>
 6 |     <Nullable>enable</Nullable>
 7 | 
 8 |     <IsPackable>false</IsPackable>
 9 |   </PropertyGroup>
10 | 
11 |   <ItemGroup>
12 |     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.12.0" />
13 |     <PackageReference Include="xunit" Version="2.9.3" />
14 |     <PackageReference Include="xunit.runner.visualstudio" Version="3.0.1">
15 |       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
16 |       <PrivateAssets>all</PrivateAssets>
17 |     </PackageReference>
18 |     <PackageReference Include="coverlet.collector" Version="6.0.4">
19 |       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
20 |       <PrivateAssets>all</PrivateAssets>
21 |     </PackageReference>
22 |   </ItemGroup>
23 | 
24 |   <ItemGroup>
25 |     <ProjectReference Include="..\src\KQLAnalyzer.csproj" />
26 |   </ItemGroup>
27 | 
28 |   <ItemGroup>
29 |     <Content Include="../environments.json">
30 |       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
31 |     </Content>
32 |     <Content Include="test_data/*.json">
33 |       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
34 |     </Content>
35 |   </ItemGroup>
36 | 
37 | </Project>
38 | 


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/test_data/custom_table.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "query": "MyTable | project Value",
 3 |     "environment": "sentinel",
 4 |     "local_data": {
 5 |         "tables": {
 6 |             "MyTable": {
 7 |                 "Timestamp": "datetime",
 8 |                 "Value": "string"
 9 |             }
10 |         }
11 |     }
12 | }


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/test_data/fileprofile.json:
--------------------------------------------------------------------------------
1 | {
2 |     "query": "print SHA1='x', Foo=123 | invoke FileProfile(SHA1)",
3 |     "environment": "m365"
4 | }


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/test_data/scalar_function.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "query": "print a=MyScalar('foo')",
 3 |     "environment": "sentinel",
 4 |     "local_data": {
 5 |         "scalar_functions": {
 6 |             "MyScalar": {
 7 |                 "output_type": "bool",
 8 |                 "arguments": [
 9 |                     {
10 |                         "name": "foo",
11 |                         "type": "string",
12 |                         "optional": true
13 |                     }
14 |                 ]
15 |             }
16 |         }
17 |     }
18 | }


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/test_data/sentinel_no_fileprofile.json:
--------------------------------------------------------------------------------
1 | {
2 |     "query": "print SHA1='x' | invoke FileProfile('SHA1')",
3 |     "environment": "sentinel"
4 | }


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/test_data/simple_query.json:
--------------------------------------------------------------------------------
1 | {
2 |     "query": "print a=true",
3 |     "environment": "sentinel"
4 | }


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/test_data/simple_query_using_default_tables.json:
--------------------------------------------------------------------------------
1 | {
2 |     "query": "AACAudit | project a=Type",
3 |     "environment": "sentinel"
4 | }


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/test_data/tabular_function.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "query": "MyFunction() | union MyFunction('foo')",
 3 |     "environment": "sentinel",
 4 |     "local_data": {
 5 |         "tabular_functions": {
 6 |             "MyFunction": {
 7 |                 "output_columns": {
 8 |                     "output_foo": "string"
 9 |                 },
10 |                 "arguments": [
11 |                     {
12 |                         "name": "foo",
13 |                         "type": "string",
14 |                         "optional": true
15 |                     }
16 |                 ]
17 |             }
18 |         }
19 |     }
20 | }


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/test_data/tabular_function_required_args.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "query": "MyFunction() | union MyFunction('foo')",
 3 |     "environment": "sentinel",
 4 |     "local_data": {
 5 |         "tabular_functions": {
 6 |             "MyFunction": {
 7 |                 "output_columns": {
 8 |                     "output_foo": "string"
 9 |                 },
10 |                 "arguments": [
11 |                     {
12 |                         "name": "foo",
13 |                         "type": "string",
14 |                         "optional": false
15 |                     }
16 |                 ]
17 |             }
18 |         }
19 |     }
20 | }


--------------------------------------------------------------------------------
/KQLAnalyzer.Tests/test_data/watchlist.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "query": "let RuleId='xyz';_GetWatchlist(strcat(strcat(RuleId,'_','foo'),'abc'))",
 3 |     "environment": "sentinel",
 4 |     "local_data": {
 5 |         "watchlists": {
 6 |             "xyz_fooabc": {
 7 |                 "foo": "string"
 8 |             }
 9 |         }
10 |     }
11 | }


--------------------------------------------------------------------------------
/KQLValidator.sln:
--------------------------------------------------------------------------------
 1 | 
 2 | Microsoft Visual Studio Solution File, Format Version 12.00
 3 | # Visual Studio Version 17
 4 | VisualStudioVersion = 17.0.31903.59
 5 | MinimumVisualStudioVersion = 10.0.40219.1
 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KQLAnalyzer", "src\KQLAnalyzer.csproj", "{E8C805FD-0ABB-46F1-87CD-15609E0531B6}"
 7 | EndProject
 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KQLAnalyzer.Tests", "KQLAnalyzer.Tests\KQLAnalyzer.Tests.csproj", "{24674704-4F73-43F2-968F-1537E40630D1}"
 9 | EndProject
10 | Global
11 | 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
12 | 		Debug|Any CPU = Debug|Any CPU
13 | 		Release|Any CPU = Release|Any CPU
14 | 	EndGlobalSection
15 | 	GlobalSection(SolutionProperties) = preSolution
16 | 		HideSolutionNode = FALSE
17 | 	EndGlobalSection
18 | 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
19 | 		{E8C805FD-0ABB-46F1-87CD-15609E0531B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
20 | 		{E8C805FD-0ABB-46F1-87CD-15609E0531B6}.Debug|Any CPU.Build.0 = Debug|Any CPU
21 | 		{E8C805FD-0ABB-46F1-87CD-15609E0531B6}.Release|Any CPU.ActiveCfg = Release|Any CPU
22 | 		{E8C805FD-0ABB-46F1-87CD-15609E0531B6}.Release|Any CPU.Build.0 = Release|Any CPU
23 | 		{24674704-4F73-43F2-968F-1537E40630D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
24 | 		{24674704-4F73-43F2-968F-1537E40630D1}.Debug|Any CPU.Build.0 = Debug|Any CPU
25 | 		{24674704-4F73-43F2-968F-1537E40630D1}.Release|Any CPU.ActiveCfg = Release|Any CPU
26 | 		{24674704-4F73-43F2-968F-1537E40630D1}.Release|Any CPU.Build.0 = Release|Any CPU
27 | 	EndGlobalSection
28 | EndGlobal
29 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # KustoQueryAnalyzer
  2 | 
  3 | This tool can be used to analyze KQL queries and provide the following information:
  4 | * Query syntax errors.
  5 | * List of output columns and their types.
  6 | * List of referenced table names.
  7 | * List of referenced column names.
  8 | * List of referenced functions.
  9 | 
 10 | ## Usage
 11 | 
 12 | The tool can be used in two different ways:
 13 | 
 14 | ### 1. As a command line tool
 15 | 
 16 | The tool can be used as a command line tool to analyze a single query.
 17 | 
 18 | ```
 19 | dotnet run --project=src/KQLAnalyzer.csproj -- --input-file <input-file>
 20 | ```
 21 | 
 22 | For example:
 23 | ```
 24 | dotnet run --project=src/KQLAnalyzer.csproj -- --input-file=query.json
 25 | ```
 26 | 
 27 | Where query.json is a file containing the query to be analyzed in JSON format, for example:
 28 | ```
 29 | {
 30 |     "query": "let RuleId=12345;SigninLogs|where UserPrincipalName in~ ((_GetWatchlist(strcat('wl','_',RuleId)) | project SPN)) | project TimeGenerated, UserPrincipalName",
 31 |     "environment": "sentinel",
 32 |     "local_data": {
 33 |         "watchlists": {
 34 |             "wl_12345": {
 35 |                 "SPN": "string"
 36 |             }
 37 |         }
 38 |     }
 39 | }
 40 | ```
 41 | 
 42 | For this example the following output is given by the tool:
 43 | ```
 44 | {
 45 |   "output_columns": {
 46 |     "TimeGenerated": "datetime",
 47 |     "UserPrincipalName": "string"
 48 |   },
 49 |   "parsing_errors": [],
 50 |   "referenced_tables": [
 51 |     "SigninLogs"
 52 |   ],
 53 |   "referenced_functions": [
 54 |     "_GetWatchlist"
 55 |   ],
 56 |   "referenced_columns": [
 57 |     "UserPrincipalName",
 58 |     "TimeGenerated"
 59 |   ],
 60 |   "elapsed_ms": 94
 61 | }
 62 | ```
 63 | 
 64 | ### 2. As a REST web service
 65 | 
 66 | The tool can also be used as a REST web service to analyze multiple queries without having to restart the executable each time.
 67 | 
 68 | ```
 69 | dotnet run --project=src/KQLAnalyzer.csproj -- --rest --bind-address=http://localhost:8000
 70 | # The bind-address parameter is optional and defaults to http://localhost:8000.
 71 | ```
 72 | 
 73 | The REST web service will be available at http://localhost:8000.
 74 | 
 75 | The following endpoints are available:
 76 | * POST /api/analyze - Analyze a query providing a query and platform.
 77 | * GET  /api/environments - List available platforms.
 78 | 
 79 | Example usage:
 80 | ```
 81 | curl -X POST -H "Content-Type: application/json" -d@query.json http://localhost:8000/api/analyze
 82 | ```
 83 | 
 84 | The input format for the POST request is the same as the input format for the command line tool.
 85 | 
 86 | ## Getting Schema Information
 87 | 
 88 | The tool can parse Microsoft documentation to get schema information for the Sentinel and M365 Defender platforms.
 89 | 
 90 | To update the the schema information based on the latest documentation, run the following command:
 91 | ```
 92 | ./update_schemas.sh
 93 | ```
 94 | 
 95 | This will produce a file called `environments.json` which contains details about the tables and built-in functions for each platform.
 96 | 
 97 | A file called `environments.json` is already included in the repository, so you don't need to run this command unless you want to update the schema information.
 98 | 
 99 | ## Input file format
100 | 
101 | The query.json file can contain the following properties:
102 | * `environment` - The platform to use for the query. This can be one of the following values: `sentinel` or `m365`.
103 | * `query` - The query to analyze.
104 | * `local_data` - A dictionary containing local data that can be used to analyze the query. This is useful if you want to analyze a query that uses custom tables or functions. The format of this property is described below.
105 | 
106 | The `local_data` property can contain the following properties:
107 | * `tables` - A list of tables and their corresponding columns that are present in the environment. This is useful if you want to analyze a query that uses custom tables. The format of this property is the same as the `tables` property in the `environments.json` file.
108 | * `scalar_functions` - A list of scalar functions that are present in the environment. A scalar function is a function that returns a single value.
109 | * `tabular_functions` - A list of tabular functions that are present in the environment. A tabular function is a function that returns a table.
110 | * `watchlists` - A list of watchlists that are present in the environment and their corresponding custom output columns.
111 | 
112 | A more complex example that provides all of these is given below:
113 | ```
114 | {
115 |     "query": "print a=MyScalar('foo')",
116 |     "environment": "sentinel",
117 |     "local_data": {
118 |         "tables": {
119 |             "MyTable": {
120 |                 "Timestamp": "datetime",
121 |                 "Value": "string"
122 |             }
123 |         },
124 |         "scalar_functions": {
125 |             "MyScalar": {
126 |                 "output_type": "bool",
127 |                 "arguments": [
128 |                     {
129 |                         "name": "foo",
130 |                         "type": "string",
131 |                         "optional": true
132 |                     }
133 |                 ]
134 |             }
135 |         },
136 |         "tabular_functions": {
137 |             "MyFunction": {
138 |                 "output_columns": {
139 |                     "output_foo": "string"
140 |                 },
141 |                 "arguments": [
142 |                     {
143 |                         "name": "foo",
144 |                         "type": "string",
145 |                         "optional": true
146 |                     }
147 |                 ]
148 |             }
149 |         },
150 |         "watchlists": {
151 |             "example": {
152 |                 "foo": "string"
153 |             }
154 |         }
155 |     }
156 | }
157 | ```
158 | 


--------------------------------------------------------------------------------
/additional_columns.json:
--------------------------------------------------------------------------------
  1 | {
  2 |   "m365": {},
  3 |   "sentinel": {
  4 |     "AzureDiagnostics": {
  5 |       "TimeGenerated": "datetime",
  6 |       "addedAccessPolicy_TenantId_g": "string",
  7 |       "addedAccessPolicy_Permissions_certificates_s": "string",
  8 |       "addedAccessPolicy_Permissions_secrets_s": "string",
  9 |       "addedAccessPolicy_Permissions_keys_s": "string",
 10 |       "identity_claim_http_schemas_xmlsoap_org_ws_2005_05_identity_claims_upn_s": "string",
 11 |       "addedAccessPolicy_ObjectId_g": "string",
 12 |       "removedAccessPolicy_ObjectId_g": "string",
 13 |       "identity_claim_http_schemas_microsoft_com_identity_claims_objectidentifier_g": "string",
 14 |       "identity_claim_oid_g": "string",
 15 |       "identity_claim_upn_s": "string"
 16 |     },
 17 |     "SecurityEvent": {
 18 |       "AccessList": "string"
 19 |     },
 20 |     "AuditLogs": {
 21 |       "TenantId": "string"
 22 |     },
 23 |     "AADSignInEventsBeta": {
 24 |       "Timestamp": "datetime"
 25 |     },
 26 |     "AADSpnSignInEventsBeta": {
 27 |       "Timestamp": "datetime"
 28 |     },
 29 |     "AlertEvidence": {
 30 |       "Timestamp": "datetime"
 31 |     },
 32 |     "AlertInfo": {
 33 |       "Timestamp": "datetime"
 34 |     },
 35 |     "CloudAppEvents": {
 36 |       "Timestamp": "datetime"
 37 |     },
 38 |     "DeviceEvents": {
 39 |       "Timestamp": "datetime"
 40 |     },
 41 |     "DeviceFileCertificateInfo": {
 42 |       "Timestamp": "datetime"
 43 |     },
 44 |     "DeviceFileEvents": {
 45 |       "Timestamp": "datetime"
 46 |     },
 47 |     "DeviceImageLoadEvents": {
 48 |       "Timestamp": "datetime"
 49 |     },
 50 |     "DeviceInfo": {
 51 |       "Timestamp": "datetime"
 52 |     },
 53 |     "DeviceLogonEvents": {
 54 |       "Timestamp": "datetime"
 55 |     },
 56 |     "DeviceNetworkEvents": {
 57 |       "Timestamp": "datetime"
 58 |     },
 59 |     "DeviceNetworkInfo": {
 60 |       "Timestamp": "datetime"
 61 |     },
 62 |     "DeviceProcessEvents": {
 63 |       "Timestamp": "datetime"
 64 |     },
 65 |     "DeviceRegistryEvents": {
 66 |       "Timestamp": "datetime"
 67 |     },
 68 |     "DeviceTvmInfoGathering": {
 69 |       "Timestamp": "datetime"
 70 |     },
 71 |     "DeviceTvmSecureConfigurationAssessment": {
 72 |       "Timestamp": "datetime"
 73 |     },
 74 |     "EmailAttachmentInfo": {
 75 |       "Timestamp": "datetime"
 76 |     },
 77 |     "EmailEvents": {
 78 |       "Timestamp": "datetime"
 79 |     },
 80 |     "EmailPostDeliveryEvents": {
 81 |       "Timestamp": "datetime"
 82 |     },
 83 |     "EmailUrlInfo": {
 84 |       "Timestamp": "datetime"
 85 |     },
 86 |     "IdentityDirectoryEvents": {
 87 |       "Timestamp": "datetime"
 88 |     },
 89 |     "IdentityLogonEvents": {
 90 |       "Timestamp": "datetime"
 91 |     },
 92 |     "IdentityQueryEvents": {
 93 |       "Timestamp": "datetime"
 94 |     },
 95 |     "UrlClickEvents": {
 96 |       "Timestamp": "datetime"
 97 |     },
 98 |     "SailPointIDN_Events_CL": {
 99 |       "TenantId": "string",
100 |       "SourceSystem": "string",
101 |       "MG": "string",
102 |       "ManagementGroupName": "string",
103 |       "TimeGenerated": "datetime",
104 |       "Computer": "string",
105 |       "RawData": "string",
106 |       "attributes_membership_before_s": "string",
107 |       "attributes_membership_after_s": "string",
108 |       "attributes_accessProfiles_after_s": "string",
109 |       "attributes_modified_before_t": "datetime",
110 |       "attributes_modified_after_t": "datetime",
111 |       "attributes_accessProfiles_before_s": "string",
112 |       "attributes_newObj_s": "string",
113 |       "attributes_processId_g": "string",
114 |       "attributes_clientId_g": "string",
115 |       "attributes_clientName_s": "string",
116 |       "attributes_creationTime_t": "datetime",
117 |       "attributes_appRefs_s": "string",
118 |       "attributes_attributeValue_g": "string",
119 |       "attributes_accountName_g": "string",
120 |       "attributes_sourceName_after_s": "string",
121 |       "attributes_sourceName_before_s": "string",
122 |       "attributes_sourceId_after_s": "string",
123 |       "attributes_type_after_s": "string",
124 |       "attributes_type_before_s": "string",
125 |       "attributes_sourceId_before_s": "string",
126 |       "actor_name_g": "string",
127 |       "ipAddress_g": "string",
128 |       "attributes_finalRecipient_s": "string",
129 |       "attributes_hostName_g": "string",
130 |       "attributes_multiValue_s": "string",
131 |       "attributes_synchronizeTo_s": "string",
132 |       "attributes_synchronizeFrom_s": "string",
133 |       "attributes_sourceName_g": "string",
134 |       "attributes_accessProfileId_g": "string",
135 |       "attributes_roleId_g": "string",
136 |       "attributes_enabled_s": "string",
137 |       "attributes_segment_Ids_s": "string",
138 |       "attributes_id_g": "string",
139 |       "attributes_segment_s": "string",
140 |       "attributes_provisioningResult_s": "string",
141 |       "attributes_previousValue_s": "string",
142 |       "attributes_identityCount_after_s": "string",
143 |       "attributes_identityCount_before_s": "string",
144 |       "attributes_addedAttributes_s": "string",
145 |       "attributes_changeset_s": "string",
146 |       "attributes_removedAttributes_s": "string",
147 |       "attributes_accountId_s": "string",
148 |       "attributes_segments_before_s": "string",
149 |       "attributes_segments_after_s": "string",
150 |       "attributes_selector_after_s": "string",
151 |       "attributes_selector_before_s": "string",
152 |       "attributes_taskResultId_g": "string",
153 |       "attributes_role_ids_s": "string",
154 |       "attributes_access_profile_ids_s": "string",
155 |       "attributes_entitlement_ids_s": "string",
156 |       "attributes_reviewerComment_s": "string",
157 |       "attributes_reviewerCommentDate_s": "string",
158 |       "attributes_requesterCommentsDate_s": "string",
159 |       "attributes_entitlementCount_after_s": "string",
160 |       "attributes_entitlementCount_before_s": "string",
161 |       "attributes_name_before_s": "string",
162 |       "attributes_description_before_s": "string",
163 |       "attributes_description_after_s": "string",
164 |       "attributes_name_after_s": "string",
165 |       "attributes_displayName_before_s": "string",
166 |       "attributes_displayName_after_s": "string",
167 |       "target_name_g": "string",
168 |       "attributes_accountSelector_before_s": "string",
169 |       "attributes_ownerId_after_g": "string",
170 |       "attributes_ownerId_before_g": "string",
171 |       "attributes_entitlements_before_s": "string",
172 |       "attributes_entitlements_after_s": "string",
173 |       "attributes_accountSelector_after_s": "string",
174 |       "attributes_owner_after_s": "string",
175 |       "attributes_owner_before_s": "string",
176 |       "attributes_info_g": "string",
177 |       "attributes_oldOwner_s": "string",
178 |       "attributes_newOwner_s": "string",
179 |       "attributes_originOrg_s": "string",
180 |       "attributes_originUsername_s": "string",
181 |       "attributes_authTenant_s": "string",
182 |       "attributes_authUserName_s": "string",
183 |       "attributes_role_s": "string",
184 |       "attributes_expirationDate_s": "string",
185 |       "attributes_errors_s": "string",
186 |       "attributes_description_s": "string",
187 |       "attributes_workitem_g": "string",
188 |       "attributes_name_s": "string",
189 |       "attributes_requestable_after_s": "string",
190 |       "attributes_revokeRequestApprovalSchemes_before_s": "string",
191 |       "attributes_requestable_before_s": "string",
192 |       "attributes_revokeRequestApprovalSchemes_after_s": "string",
193 |       "attributes_comment_s": "string",
194 |       "attributes_segmentId_g": "string",
195 |       "attributes_object_type_s": "string",
196 |       "attributes_deniedCommentsRequired_after_s": "string",
197 |       "attributes_disabled_before_s": "string",
198 |       "attributes_disabled_after_s": "string",
199 |       "attributes_requestCommentsRequired_after_s": "string",
200 |       "attributes_approvalSchemes_before_s": "string",
201 |       "attributes_deniedCommentsRequired_before_s": "string",
202 |       "attributes_requestCommentsRequired_before_s": "string",
203 |       "attributes_approvalSchemes_after_s": "string",
204 |       "attributes_accessProfileIds_after_s": "string",
205 |       "attributes_accessProfileIds_before_s": "string",
206 |       "attributes_requestedAppName_s": "string",
207 |       "attributes_requestedAppRoleId_g": "string",
208 |       "attributes_requestedAppId_s": "string",
209 |       "attributes_accountUuid_g": "string",
210 |       "attributes_oldState_s": "string",
211 |       "attributes_newState_s": "string",
212 |       "attributes_objectType_s": "string",
213 |       "attributes_customerName_s": "string",
214 |       "attributes_customerId_g": "string",
215 |       "attributes_qualifier_s": "string",
216 |       "attributes_dagId_s": "string",
217 |       "attributes_details_s": "string",
218 |       "attributes_aggregationId_g": "string",
219 |       "attributes_approvalSchemesToOwnerMap_s": "string",
220 |       "attributes_preventativeSODResultsJSON_s": "string",
221 |       "attributes_interface_s": "string",
222 |       "attributes_operation_s": "string",
223 |       "attributes_approvalSchemesList_s": "string",
224 |       "attributes_flow_s": "string",
225 |       "attributes_IdnAccessRequestAttributes_s": "string",
226 |       "attributes_accountName_s": "string",
227 |       "attributes_attributeName_s": "string",
228 |       "attributes_attributeValue_s": "string",
229 |       "attributes_accountUuid_s": "string",
230 |       "attributes_cloudAppName_s": "string",
231 |       "attributes_appId_g": "string",
232 |       "attributes_sourceId_s": "string",
233 |       "attributes_removeDate_t": "datetime",
234 |       "details_s": "string",
235 |       "attributes_accountActivityId_g": "string",
236 |       "attributes_accessItemType_s": "string",
237 |       "attributes_comments_s": "string",
238 |       "attributes_accessItemId_g": "string",
239 |       "attributes_accessItemName_s": "string",
240 |       "IPAddress": "string",
241 |       "details_g": "string",
242 |       "attributes_pod_s": "string",
243 |       "attributes_org_s": "string",
244 |       "attributes_info_s": "string",
245 |       "id_s": "string",
246 |       "attributes_sourceName_s": "string",
247 |       "org_s": "string",
248 |       "pod_s": "string",
249 |       "created_t": "datetime",
250 |       "id_g": "string",
251 |       "action_s": "string",
252 |       "type_s": "string",
253 |       "actor_name_s": "string",
254 |       "target_name_s": "string",
255 |       "stack_s": "string",
256 |       "trackingNumber_g": "string",
257 |       "attributes_hostName_s": "string",
258 |       "attributes_userId_s": "string",
259 |       "attributes_scope_s": "string",
260 |       "objects_s": "string",
261 |       "operation_s": "string",
262 |       "status_s": "string",
263 |       "technicalName_s": "string",
264 |       "name_s": "string",
265 |       "synced_t": "datetime",
266 |       "_type_s": "string",
267 |       "_version_s": "string",
268 |       "Type": "string",
269 |       "_ResourceId": "string"
270 |     },
271 |     "ABAPAuditLog_CL": {
272 |       "TenantId": "string",
273 |       "SourceSystem": "string",
274 |       "MG": "string",
275 |       "ManagementGroupName": "string",
276 |       "TimeGenerated": "datetime",
277 |       "Computer": "string",
278 |       "RawData": "string",
279 |       "SystemID_s": "string",
280 |       "Instance_s": "string",
281 |       "MessageText_s": "string",
282 |       "MessageClass_s": "string",
283 |       "MessageID_s": "string",
284 |       "AlertSeverity_d": "real",
285 |       "ClientID_s": "string",
286 |       "User_s": "string",
287 |       "TransactionCode_s": "string",
288 |       "ABAPProgramName_s": "string",
289 |       "AuditClassID_d": "real",
290 |       "AlertSeverityText_s": "string",
291 |       "TerminalIPv6_s": "string",
292 |       "Variable1_s": "string",
293 |       "Variable2_s": "string",
294 |       "Variable3_s": "string",
295 |       "Variable4_s": "string",
296 |       "SAPProcesType_s": "string",
297 |       "SAPWPName_s": "string",
298 |       "Email_s": "string",
299 |       "SystemNumber_s": "string",
300 |       "Host_s": "string",
301 |       "Type": "string",
302 |       "_ResourceId": "string"
303 |     }
304 |   }
305 | }
306 | 


--------------------------------------------------------------------------------
/get_schemas.py:
--------------------------------------------------------------------------------
  1 | import glob
  2 | import json
  3 | import os
  4 | import textwrap
  5 | import re
  6 | 
  7 | valid_types = {'datetime': True, 'string': True, 'int': True, 'boolean': True, 'long': True, 'bool': True, 'dynamic': True, 'real': True, 'guid': True, 'double': True}
  8 | 
  9 | # Extract tables from markdown files in Microsoft documentation.
 10 | def get_table_details(fn, base_dir):
 11 |     inside_table = False
 12 |     table_name = None
 13 |     details = {}
 14 |     data = open(fn).read()
 15 |     # Parse [!INCLUDE [awscloudtrail](../includes/awscloudtrail-include.md)]
 16 |     for include_fn in re.findall(r'\[!INCLUDE \[.*?\]\((.*?)\)\]', data):
 17 |         if 'reusable-content' in include_fn:
 18 |             print(include_fn)
 19 |             exit()
 20 |         include_path = os.path.abspath(os.path.join(os.path.dirname(fn), include_fn))
 21 |         parsed_dir = os.path.dirname(os.path.dirname(include_path)) + os.sep
 22 |         if not parsed_dir.startswith(base_dir + os.sep):
 23 |             raise Exception(f"Include path {parsed_dir} is not in {base_dir}")
 24 |         data += open(include_path).read() + '\n'
 25 | 
 26 |     for line in data.splitlines():
 27 |         line = line.strip()
 28 |         if not line:
 29 |             continue
 30 |         line = line.replace('`','')
 31 |         if not table_name and line.startswith('# '):
 32 |             table_name = line.split()[1]
 33 |         if (
 34 |             line.lower().startswith('## columns')
 35 |             or line.lower().startswith('| column name')
 36 |             or line.lower().startswith('|column name')
 37 |             ):
 38 |             inside_table = True
 39 |             continue
 40 |         # if not line.startswith('|'):
 41 |         #     inside_table = False
 42 |         if line.startswith('#'):
 43 |             inside_table = False
 44 |         if not inside_table or not line.startswith('|'):
 45 |             continue
 46 |         column_details = line.replace(' ','').replace('\t','').split('|')
 47 |         if len(column_details) < 4:
 48 |             continue
 49 |         column_name = column_details[1]
 50 |         column_type = column_details[2].lower()
 51 |         if column_type == 'bigint':
 52 |             column_type = 'long'
 53 |         if column_type == 'list':
 54 |             column_type = 'string' # some tables refer to non-existing type 'list'
 55 |         if column_name == 'Column' or column_name.startswith('--') or not column_name:
 56 |             continue
 57 |         if not column_type in valid_types:
 58 |             raise Exception(f"{column_type} is not a valid column type")
 59 |         details[column_name] = column_type
 60 |     return table_name, details
 61 | 
 62 | def merge_additional_columns(tables, env_name):
 63 |     additional_columns = json.load(open('additional_columns.json'))[env_name]
 64 |     for table_name, extra_fields in additional_columns.items():
 65 |         if table_name not in tables:
 66 |             tables[table_name] = {}
 67 |         for field_name, field_type in extra_fields.items():
 68 |             tables[table_name][field_name] = field_type
 69 | 
 70 | environments = {
 71 |     'm365': {
 72 |          'dir_name': 'defender-docs/defender-xdr',
 73 |          'base_dir': 'defender-docs',
 74 |          'glob': '*-table.md',
 75 |          'help': textwrap.dedent("""
 76 |             git clone --depth=1 https://github.com/MicrosoftDocs/defender-docs
 77 |         """),
 78 |         'magic_functions': [
 79 |             'FileProfile',
 80 |             'DeviceFromIP'
 81 |         ]
 82 |     },
 83 |     'sentinel': {
 84 |        'dir_name': 'azure-reference-other/azure-monitor-ref/tables',
 85 |        'base_dir': 'azure-reference-other',
 86 |          'glob': '*.md',
 87 |        'help': textwrap.dedent("""
 88 |             git clone https://github.com/MicrosoftDocs/azure-reference-other ; git checkout 97f4433e37c4a95922407dcc4c3014c4badb6881
 89 |         """),
 90 |     }
 91 | }
 92 | 
 93 | def main():
 94 |     environment_details = {}
 95 |     for env_name, env_details in environments.items():
 96 |         if not os.path.exists(env_details['dir_name']):
 97 |             print(f"ERROR: {env_details['dir_name']} does not exist. To create it, run:\n{env_details['help'].strip()}")
 98 |             exit(1)
 99 |         base_dir = os.path.abspath(env_details['base_dir'])
100 |         tables = {}
101 |         glob_pattern = os.path.join(env_details['dir_name'], env_details['glob'])
102 |         for table_fn in sorted(glob.glob(glob_pattern)):
103 |             table_name, details = get_table_details(table_fn, base_dir)
104 |             tables[table_name] = details
105 |         merge_additional_columns(tables, env_name)
106 |         details = dict(tables=tables, magic_functions=env_details.get('magic_functions', []))
107 |         environment_details[env_name] = details
108 |     print(json.dumps(environment_details, indent=2))
109 | 
110 | if __name__ == '__main__':
111 |     main()
112 | 


--------------------------------------------------------------------------------
/src/EnvironmentDefinition.cs:
--------------------------------------------------------------------------------
 1 | using System.Text.Json;
 2 | using System.Text.Json.Serialization;
 3 | using Kusto.Language;
 4 | using Kusto.Language.Symbols;
 5 | 
 6 | public class EnvironmentDefinition
 7 | {
 8 |     private static readonly FunctionSymbol FileProfileFunction =
 9 |         // Create a function that has the same return type as FileProfile
10 |         // Copies all the columns from the existing table and adds the new ones as specified by Microsoft.
11 |         // This is a workaround for the fact that the FileProfile function is not available in the Kusto API.
12 |         // The join is there to get the same behavior as the FileProfile function where if a column
13 |         // named GlobalPrevalence or any other output column is already present in the table,
14 |         // it will not be overwritten but a new  column called GlobalPrevalence1 will be added.
15 |         new (
16 |             "FileProfile",
17 |             "(T:(*), x:string='',y:string='')",
18 |             """
19 |             {
20 |                 (T | extend _TmpJoinKey=123)
21 |                 | join (
22 |                     print SHA1='', SHA256='', MD5='', FileSize=0, GlobalPrevalence=0, GlobalFirstSeen=now(),
23 |                     GlobalLastSeen=now(), Signer='', Issuer='', SignerHash='', IsCertificateValid=false,
24 |                     IsRootSignerMicrosoft=false, SignatureState='', IsExecutable=false, ThreatName='',
25 |                     Publisher='', SoftwareName='', ProfileAvailability='',_TmpJoinKey=123
26 |                 ) on _TmpJoinKey | project-away _TmpJoinKey, _TmpJoinKey1
27 |             }
28 |             """
29 |         );
30 | 
31 |     private static readonly FunctionSymbol DeviceFromIPFunction =
32 |         new (
33 |             "DeviceFromIP",
34 |             "(T:(*), x:string='',y:datetime='')",
35 |             """
36 |             {
37 |                 (T | extend _TmpJoinKey=123)
38 |                 | join (
39 |                     print IP='', DeviceId='', _TmpJoinKey=123
40 |                 ) on _TmpJoinKey | project-away _TmpJoinKey, _TmpJoinKey1
41 |             }
42 |             """
43 |         );
44 | 
45 |     public EnvironmentDefinition()
46 |     {
47 |         this.TableDetails = new Dictionary<string, TableDetails>();
48 |         this.MagicFunctions = new List<string>();
49 |     }
50 | 
51 |     [JsonPropertyName("tables")]
52 |     public Dictionary<string, TableDetails> TableDetails { get; set; }
53 | 
54 |     [JsonPropertyName("magic_functions")]
55 |     public List<string> MagicFunctions { get; set; }
56 | 
57 |     public GlobalState ToGlobalState()
58 |     {
59 |         List<Symbol> dbMembers = new List<Symbol>();
60 | 
61 |         foreach (var table in this.TableDetails)
62 |         {
63 |             List<ColumnSymbol> columns = table.Value
64 |                 .Select(column =>
65 |                 {
66 |                     if (ScalarTypes.GetSymbol(column.Value) == null)
67 |                     {
68 |                         throw new Exception(
69 |                             $"Unknown type {column.Value} for column {column.Key} in table {table.Key}"
70 |                         );
71 |                     }
72 | 
73 |                     return new ColumnSymbol(column.Key, ScalarTypes.GetSymbol(column.Value));
74 |                 })
75 |                 .ToList();
76 |             dbMembers.Add(new TableSymbol(table.Key, columns));
77 |         }
78 | 
79 |         if (this.MagicFunctions.Contains("FileProfile"))
80 |         {
81 |             dbMembers.Add(FileProfileFunction);
82 |         }
83 | 
84 |         if (this.MagicFunctions.Contains("DeviceFromIP"))
85 |         {
86 |             dbMembers.Add(DeviceFromIPFunction);
87 |         }
88 | 
89 |         return GlobalState.Default.WithDatabase(new DatabaseSymbol("db", dbMembers));
90 |     }
91 | }
92 | 


--------------------------------------------------------------------------------
/src/KQLAnalyzer.csproj:
--------------------------------------------------------------------------------
 1 | <Project Sdk="Microsoft.NET.Sdk.Web">
 2 | 
 3 |   <PropertyGroup>
 4 |     <OutputType>Exe</OutputType>
 5 |     <TargetFramework>net8.0</TargetFramework>
 6 |     <ImplicitUsings>enable</ImplicitUsings>
 7 |     <Nullable>enable</Nullable>
 8 |     <NoWarn>1591,SA1633,SA1600,RCS0044,RCS1176,RCS1010,RCS1177,SA1111,SA1009,SA1118,SA1200,CA1050,SA1515,IDE0090,SA1413,SA1502</NoWarn>
 9 |   </PropertyGroup>
10 | 
11 |   <ItemGroup>
12 |     <PackageReference Include="Microsoft.Azure.Kusto.Language" Version="11.2.2" />
13 |     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118">
14 |       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
15 |       <PrivateAssets>all</PrivateAssets>
16 |     </PackageReference>
17 |     <PackageReference Include="System.CommandLine.DragonFruit" Version="0.4.0-alpha.22272.1" />
18 |   </ItemGroup>
19 | </Project>
20 | 


--------------------------------------------------------------------------------
/src/KQLAnalyzerRESTService.cs:
--------------------------------------------------------------------------------
 1 | namespace KQLAnalyzer
 2 | {
 3 |     public class KQLAnalyzerRESTService
 4 |     {
 5 |         public static IResult Analyze(AnalyzeRequest data, KQLEnvironments kqlEnvironments)
 6 |         {
 7 |             // Check if environment is in KqlEnvironment.Environments
 8 |             if (!kqlEnvironments.ContainsKey(data.Environment))
 9 |             {
10 |                 return Results.NotFound("Environment not found");
11 |             }
12 | 
13 |             var globals = kqlEnvironments[data.Environment].ToGlobalState();
14 |             var results = KustoAnalyzer.AnalyzeQuery(data.Query, globals, data.LocalData);
15 |             return Results.Ok(results);
16 |         }
17 | 
18 |         public static void LaunchRestServer(string bindAddress, KQLEnvironments kqlEnvironments)
19 |         {
20 |             var app = WebApplication.Create();
21 |             app.MapGet("/api/environments", () => kqlEnvironments.Keys);
22 |             app.MapPost("/api/analyze", (AnalyzeRequest data) => Analyze(data, kqlEnvironments));
23 |             app.Run(bindAddress);
24 |         }
25 |     }
26 | }
27 | 


--------------------------------------------------------------------------------
/src/KustoAnalyzer.cs:
--------------------------------------------------------------------------------
  1 | using System.Text.Json;
  2 | using System.Text.RegularExpressions;
  3 | using Kusto.Language;
  4 | using Kusto.Language.Symbols;
  5 | using Kusto.Language.Syntax;
  6 | 
  7 | namespace KQLAnalyzer
  8 | {
  9 |     public static class KustoAnalyzer
 10 |     {
 11 |         // This function was taken from
 12 |         // https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/readme.md
 13 |         public static HashSet<TableSymbol> GetDatabaseTables(KustoCode code)
 14 |         {
 15 |             var tables = new HashSet<TableSymbol>();
 16 | 
 17 |             SyntaxElement.WalkNodes(
 18 |                 code.Syntax,
 19 |                 n =>
 20 |                 {
 21 |                     if (n.ReferencedSymbol is TableSymbol t && code.Globals.IsDatabaseTable(t))
 22 |                     {
 23 |                         tables.Add(t);
 24 |                     }
 25 |                     else if (
 26 |                         n is Expression e
 27 |                         && e.ResultType is TableSymbol ts
 28 |                         && code.Globals.IsDatabaseTable(ts)
 29 |                     )
 30 |                     {
 31 |                         tables.Add(ts);
 32 |                     }
 33 |                 }
 34 |             );
 35 | 
 36 |             return tables;
 37 |         }
 38 | 
 39 |         public static HashSet<FunctionSymbol> GetDatabaseFunctions(KustoCode code)
 40 |         {
 41 |             var functions = new HashSet<FunctionSymbol>();
 42 | 
 43 |             SyntaxElement.WalkNodes(
 44 |                 code.Syntax,
 45 |                 n =>
 46 |                 {
 47 |                     if (
 48 |                         n.ReferencedSymbol is FunctionSymbol t && code.Globals.IsDatabaseFunction(t)
 49 |                     )
 50 |                     {
 51 |                         functions.Add(t);
 52 |                     }
 53 |                     else if (
 54 |                         n is Expression e
 55 |                         && e.ResultType is FunctionSymbol ts
 56 |                         && code.Globals.IsDatabaseFunction(ts)
 57 |                     )
 58 |                     {
 59 |                         functions.Add(ts);
 60 |                     }
 61 |                 }
 62 |             );
 63 | 
 64 |             return functions;
 65 |         }
 66 | 
 67 |         // This function was taken from
 68 |         // https://github.com/microsoft/Kusto-Query-Language/blob/master/src/Kusto.Language/readme.md
 69 |         public static HashSet<ColumnSymbol> GetDatabaseTableColumns(KustoCode code)
 70 |         {
 71 |             var columns = new HashSet<ColumnSymbol>();
 72 |             GatherColumns(code.Syntax);
 73 |             return columns;
 74 | 
 75 |             void GatherColumns(SyntaxNode root)
 76 |             {
 77 |                 SyntaxElement.WalkNodes(
 78 |                     root,
 79 |                     fnBefore: n =>
 80 |                     {
 81 |                         if (
 82 |                             n.ReferencedSymbol is ColumnSymbol c && code.Globals.GetTable(c) != null
 83 |                         )
 84 |                         {
 85 |                             columns.Add(c);
 86 |                         }
 87 |                         else if (n.GetCalledFunctionBody() is SyntaxNode body)
 88 |                         {
 89 |                             GatherColumns(body);
 90 |                         }
 91 |                     },
 92 |                     fnDescend: n =>
 93 |                         // skip descending into function declarations since their bodies will be examined by the code above
 94 |                         !(n is FunctionDeclaration)
 95 |                 );
 96 |             }
 97 |         }
 98 | 
 99 |         // Helper function that will resolve an expression to a string.
100 |         // It supports constants as well as applications of strcat with constant
101 |         // arguments.
102 |         // It won't work for more complex expressions that call other functions since the
103 |         // Kusto.Language analyzer doesn't have an implementation for those functions.
104 |         // The reason for supporting strcat is that there are many queries that for example
105 |         // do something like this:
106 |         // let RuleName='MyRule';
107 |         // _GetWatchlist(strcat("Watchlist_", RuleName))
108 |         // In theory other functions could be supported as well but they would have to
109 |         // be re-written in C#.
110 |         public static string ResolveStringExpression(Expression expr)
111 |         {
112 |             if (expr == null)
113 |             {
114 |                 return string.Empty;
115 |             }
116 | 
117 |             if (expr.ConstantValue != null)
118 |             {
119 |                 return expr.ConstantValue.ToString() ?? string.Empty;
120 |             }
121 | 
122 |             if (expr is FunctionCallExpression fce)
123 |             {
124 |                 // We will resolve strcat calls here, since they are commonly
125 |                 // used to build up strings and are not resolved by the Kusto analyzer itself.
126 |                 if (fce.Name.ToString() == "strcat")
127 |                 {
128 |                     return string.Join(
129 |                         string.Empty,
130 |                         fce.ArgumentList.Expressions
131 |                             .Select(e => ResolveStringExpression(e.Element))
132 |                             .ToList()
133 |                     );
134 |                 }
135 |             }
136 | 
137 |             return string.Empty;
138 |         }
139 | 
140 |         // The GetWatchlist function uses bag_unpack internally to dynamically add columns to the output.
141 |         public static FunctionSymbol GetWatchlist(Dictionary<string, WatchlistDetails> watchlists)
142 |         {
143 |             return new FunctionSymbol(
144 |                 "_GetWatchlist",
145 |                 context =>
146 |                 {
147 |                     var watchlistAlias = ResolveStringExpression(
148 |                         context.GetArgument("watchlistAlias")
149 |                     );
150 |                     var returnedColumns = new List<ColumnSymbol>
151 |                     {
152 |                         new ColumnSymbol("_DTItemId", ScalarTypes.String),
153 |                         new ColumnSymbol("LastUpdatedTimeUTC", ScalarTypes.DateTime),
154 |                         new ColumnSymbol("SearchKey", ScalarTypes.String),
155 |                         new ColumnSymbol("WatchlistItem", ScalarTypes.Dynamic),
156 |                     };
157 |                     if (
158 |                         watchlistAlias != null
159 |                         && watchlists != null
160 |                         && watchlists.ContainsKey(watchlistAlias)
161 |                     )
162 |                     {
163 |                         returnedColumns = returnedColumns
164 |                             .Concat(
165 |                                 watchlists[watchlistAlias]
166 |                                     .Select(
167 |                                         c => new ColumnSymbol(c.Key, ScalarTypes.GetSymbol(c.Value))
168 |                                     )
169 |                                     .ToList()
170 |                             )
171 |                             .ToList();
172 |                     }
173 | 
174 |                     return new TableSymbol(returnedColumns).WithInheritableProperties(
175 |                         context.RowScope
176 |                     );
177 |                 },
178 |                 Tabularity.Tabular,
179 |                 new Parameter("watchlistAlias", ScalarTypes.String)
180 |             );
181 |         }
182 | 
183 |         public static AnalyzeResults AnalyzeQuery(string query, GlobalState globals, LocalData localData)
184 |         {
185 |             // Keep track of how long it takes to analyze the query.
186 |             var watch = System.Diagnostics.Stopwatch.StartNew();
187 |             var myGlobals = globals;
188 | 
189 |             // The FileProfile function is special in that it takes a string as a parameter,
190 |             // but the parameter is not quoted. It appears that M365 also pre-processes queries
191 |             // that contain this function to magically add quotes around the first parameter.
192 |             if (globals.Database.Functions.Any(f => f.Name == "FileProfile"))
193 |             {
194 |                 // Regex to quote the first parameter of FileProfile if it's not already quoted.
195 |                 query = Regex.Replace(
196 |                     query,
197 |                     @"(invoke\s+FileProfile\(\s*)([^\',]+)([,)])",
198 |                     "$1'$2'$3"
199 |                 );
200 |             }
201 | 
202 |             if (localData?.Watchlists != null)
203 |             {
204 |                 var customWatchlists = new List<FunctionSymbol>()
205 |                 {
206 |                     GetWatchlist(localData.Watchlists)
207 |                 };
208 | 
209 |                 myGlobals = myGlobals.WithDatabase(
210 |                     myGlobals.Database.WithMembers(
211 |                         myGlobals.Database.Members.Concat(customWatchlists)
212 |                     )
213 |                 );
214 |             }
215 | 
216 |             if (localData?.Tables != null)
217 |             {
218 |                 var customTables = GetTables(localData.Tables);
219 |                 myGlobals = myGlobals.WithDatabase(
220 |                     myGlobals.Database.WithMembers(myGlobals.Database.Members.Concat(customTables))
221 |                 );
222 |             }
223 | 
224 |             if (localData?.TabularFunctions != null)
225 |             {
226 |                 var customFunctions = GetTabularFunctions(localData.TabularFunctions);
227 |                 myGlobals = myGlobals.WithDatabase(
228 |                     myGlobals.Database.WithMembers(
229 |                         myGlobals.Database.Members.Concat(customFunctions)
230 |                     )
231 |                 );
232 |             }
233 | 
234 |             if (localData?.ScalarFunctions != null)
235 |             {
236 |                 var customFunctions = GetScalarFunctions(localData.ScalarFunctions);
237 |                 myGlobals = myGlobals.WithDatabase(
238 |                     myGlobals.Database.WithMembers(
239 |                         myGlobals.Database.Members.Concat(customFunctions)
240 |                     )
241 |                 );
242 |             }
243 | 
244 |             var queryResults = new AnalyzeResults();
245 | 
246 |             var code = KustoCode.ParseAndAnalyze(query, myGlobals);
247 | 
248 |             queryResults.ParsingErrors = code.GetDiagnostics().ToList();
249 |             queryResults.ReferencedTables = GetDatabaseTables(code).Select(t => t.Name).ToList();
250 |             queryResults.ReferencedFunctions = GetDatabaseFunctions(code)
251 |                 .Select(t => t.Name)
252 |                 .ToList();
253 |             queryResults.ReferencedColumns = GetDatabaseTableColumns(code)
254 |                 .Select(t => t.Name)
255 |                 .ToList();
256 |             if (code.ResultType != null)
257 |             {
258 |                 queryResults.OutputColumns = code.ResultType.Members
259 |                     .OfType<ColumnSymbol>()
260 |                     .ToDictionary(c => c.Name, c => c.Type.Name);
261 |             }
262 | 
263 |             watch.Stop();
264 |             queryResults.ElapsedMs = watch.ElapsedMilliseconds;
265 |             return queryResults;
266 |         }
267 | 
268 |         private static List<FunctionSymbol> GetScalarFunctions(
269 |             Dictionary<string, ScalarFunctionDetails> functions
270 |         )
271 |         {
272 |             var functionSymbols = new List<FunctionSymbol>();
273 |             foreach (var function in functions)
274 |             {
275 |                 var parameters = function.Value.Arguments.Select(
276 |                     p =>
277 |                         new Parameter(
278 |                             p.Name,
279 |                             ScalarTypes.GetSymbol(p.Type),
280 |                             minOccurring: p.Optional ? 0 : 1
281 |                         )
282 |                 );
283 |                 var functionSymbol = new FunctionSymbol(
284 |                     function.Key,
285 |                     ScalarTypes.GetSymbol(function.Value.OutputType),
286 |                     parameters.ToArray()
287 |                 );
288 |                 functionSymbols.Add(functionSymbol);
289 |             }
290 | 
291 |             return functionSymbols;
292 |         }
293 | 
294 |         private static List<FunctionSymbol> GetTabularFunctions(
295 |             Dictionary<string, TabularFunctionDetails> functions
296 |         )
297 |         {
298 |             var functionSymbols = new List<FunctionSymbol>();
299 |             foreach (var function in functions)
300 |             {
301 |                 var parameters = function.Value.Arguments.Select(
302 |                     p =>
303 |                         new Parameter(
304 |                             p.Name,
305 |                             ScalarTypes.GetSymbol(p.Type),
306 |                             minOccurring: p.Optional ? 0 : 1
307 |                         )
308 |                 );
309 |                 var functionSymbol = new FunctionSymbol(
310 |                     function.Key,
311 |                     context =>
312 |                     {
313 |                         var returnedColumns = function.Value.OutputColumns.Select(
314 |                             c => new ColumnSymbol(c.Key, ScalarTypes.GetSymbol(c.Value))
315 |                         );
316 |                         return new TableSymbol(returnedColumns).WithInheritableProperties(
317 |                             context.RowScope
318 |                         );
319 |                     },
320 |                     Tabularity.Tabular,
321 |                     parameters.ToArray()
322 |                 );
323 |                 functionSymbols.Add(functionSymbol);
324 |             }
325 | 
326 |             return functionSymbols;
327 |         }
328 | 
329 |         private static List<TableSymbol> GetTables(Dictionary<string, TableDetails> tables)
330 |         {
331 |             var tableSymbols = new List<TableSymbol>();
332 |             foreach (var table in tables)
333 |             {
334 |                 var columns = table.Value.Select(
335 |                     c => new ColumnSymbol(c.Key, ScalarTypes.GetSymbol(c.Value))
336 |                 );
337 |                 var tableSymbol = new TableSymbol(table.Key, columns);
338 |                 tableSymbols.Add(tableSymbol);
339 |             }
340 | 
341 |             return tableSymbols;
342 |         }
343 |     }
344 | }
345 | 


--------------------------------------------------------------------------------
/src/Models/AnalyzeRequest.cs:
--------------------------------------------------------------------------------
 1 | using System.Text.Json.Serialization;
 2 | 
 3 | namespace KQLAnalyzer
 4 | {
 5 |     public class AnalyzeRequest
 6 |     {
 7 |         public AnalyzeRequest()
 8 |         {
 9 |             this.Query = string.Empty;
10 |             this.Environment = string.Empty;
11 |             this.LocalData = new LocalData();
12 |         }
13 | 
14 |         [JsonPropertyName("environment")]
15 |         public string Environment { get; set; }
16 | 
17 |         [JsonPropertyName("query")]
18 |         public string Query { get; set; }
19 | 
20 |         [JsonPropertyName("local_data")]
21 |         public LocalData LocalData { get; set; }
22 |     }
23 | }
24 | 


--------------------------------------------------------------------------------
/src/Models/AnalyzeResults.cs:
--------------------------------------------------------------------------------
 1 | using System.Text.Json.Serialization;
 2 | using Kusto.Language;
 3 | 
 4 | public class AnalyzeResults
 5 | {
 6 |     public AnalyzeResults()
 7 |     {
 8 |         this.OutputColumns = new Dictionary<string, string>();
 9 |         this.ParsingErrors = new List<Diagnostic>();
10 |         this.ElapsedMs = 0;
11 |         this.ReferencedTables = new List<string>();
12 |         this.ReferencedFunctions = new List<string>();
13 |         this.ReferencedColumns = new List<string>();
14 |     }
15 | 
16 |     [JsonPropertyName("output_columns")]
17 |     public Dictionary<string, string> OutputColumns { get; set; }
18 | 
19 |     [JsonPropertyName("parsing_errors")]
20 |     public List<Diagnostic> ParsingErrors { get; set; }
21 | 
22 |     [JsonPropertyName("referenced_tables")]
23 |     public List<string> ReferencedTables { get; set; }
24 | 
25 |     [JsonPropertyName("referenced_functions")]
26 |     public List<string> ReferencedFunctions { get; set; }
27 | 
28 |     [JsonPropertyName("referenced_columns")]
29 |     public List<string> ReferencedColumns { get; set; }
30 | 
31 |     [JsonPropertyName("elapsed_ms")]
32 |     public long ElapsedMs { get; set; }
33 | }
34 | 


--------------------------------------------------------------------------------
/src/Models/ArgumentDetails.cs:
--------------------------------------------------------------------------------
 1 | using System.Text.Json.Serialization;
 2 | 
 3 | public class ArgumentDetails
 4 | {
 5 |     [JsonPropertyName("name")]
 6 |     public string? Name { get; set; }
 7 | 
 8 |     [JsonPropertyName("type")]
 9 |     public string? Type { get; set; }
10 | 
11 |     [JsonPropertyName("optional")]
12 |     public bool Optional { get; set; }
13 | }
14 | 


--------------------------------------------------------------------------------
/src/Models/KQLEnvironments.cs:
--------------------------------------------------------------------------------
1 | namespace KQLAnalyzer { }
2 | 
3 | public class KQLEnvironments : Dictionary<string, EnvironmentDefinition> { }
4 | 


--------------------------------------------------------------------------------
/src/Models/LocalData.cs:
--------------------------------------------------------------------------------
 1 | using System.Text.Json.Serialization;
 2 | 
 3 | public class LocalData
 4 | {
 5 |     public LocalData()
 6 |     {
 7 |         this.Watchlists = new Dictionary<string, WatchlistDetails>();
 8 |         this.Tables = new Dictionary<string, TableDetails>();
 9 |         this.TabularFunctions = new Dictionary<string, TabularFunctionDetails>();
10 |         this.ScalarFunctions = new Dictionary<string, ScalarFunctionDetails>();
11 |     }
12 | 
13 |     [JsonPropertyName("watchlists")]
14 |     public Dictionary<string, WatchlistDetails> Watchlists { get; set; }
15 | 
16 |     [JsonPropertyName("tables")]
17 |     public Dictionary<string, TableDetails> Tables { get; set; }
18 | 
19 |     [JsonPropertyName("tabular_functions")]
20 |     public Dictionary<string, TabularFunctionDetails> TabularFunctions { get; set; }
21 | 
22 |     [JsonPropertyName("scalar_functions")]
23 |     public Dictionary<string, ScalarFunctionDetails> ScalarFunctions { get; set; }
24 | }
25 | 


--------------------------------------------------------------------------------
/src/Models/ScalarFunctionDetails.cs:
--------------------------------------------------------------------------------
 1 | using System.Text.Json.Serialization;
 2 | 
 3 | public class ScalarFunctionDetails
 4 | {
 5 |     public ScalarFunctionDetails()
 6 |     {
 7 |         this.OutputType = string.Empty;
 8 |         this.Arguments = new List<ArgumentDetails>();
 9 |     }
10 | 
11 |     [JsonPropertyName("output_type")]
12 |     public string? OutputType { get; set; }
13 | 
14 |     [JsonPropertyName("arguments")]
15 |     public List<ArgumentDetails> Arguments { get; set; }
16 | }
17 | 


--------------------------------------------------------------------------------
/src/Models/TableDetails.cs:
--------------------------------------------------------------------------------
1 | public class TableDetails : Dictionary<string, string> { }
2 | 


--------------------------------------------------------------------------------
/src/Models/TabularFunctionDetails.cs:
--------------------------------------------------------------------------------
 1 | using System.Text.Json.Serialization;
 2 | 
 3 | public class TabularFunctionDetails
 4 | {
 5 |     public TabularFunctionDetails()
 6 |     {
 7 |         this.OutputColumns = new TableDetails();
 8 |         this.Arguments = new List<ArgumentDetails>();
 9 |     }
10 | 
11 |     [JsonPropertyName("output_columns")]
12 |     public TableDetails OutputColumns { get; set; }
13 | 
14 |     [JsonPropertyName("arguments")]
15 |     public List<ArgumentDetails> Arguments { get; set; }
16 | }
17 | 


--------------------------------------------------------------------------------
/src/Models/WatchlistDetails.cs:
--------------------------------------------------------------------------------
1 | public class WatchlistDetails : Dictionary<string, string> { }
2 | 


--------------------------------------------------------------------------------
/src/Program.cs:
--------------------------------------------------------------------------------
 1 | using System.Text.Json;
 2 | using KQLAnalyzer;
 3 | 
 4 | public class Program
 5 | {
 6 |     /// <summary>
 7 |     /// KustoQueryAnalyzer can analyze a Kusto query and returns metadata about the query.
 8 |     /// </summary>
 9 |     /// <param name="inputFile">Analyze query from JSON file.</param>
10 |     /// <param name="environmentsFile">Environment configuration file to use. Defaults to ../environments.json.</param>
11 |     /// <param name="rest">Start a REST server to listen for requests.</param>
12 |     /// <param name="bindAddress">HTTP bind address to use in format http://host:port.</param>
13 | #pragma warning disable 8625
14 |     public static void Main(
15 |         FileInfo inputFile = null,
16 |         FileInfo environmentsFile = null,
17 |         bool rest = false,
18 |         string bindAddress = "http://localhost:8000"
19 |     )
20 | #pragma warning restore 8625
21 |     {
22 |         environmentsFile = environmentsFile ?? new FileInfo(Path.Join("..", "environments.json"));
23 |         var kqlEnvironments = new KQLEnvironments();
24 |         try
25 |         {
26 |             kqlEnvironments = JsonSerializer.Deserialize<KQLEnvironments>(
27 |                 File.ReadAllText(environmentsFile.FullName)
28 |             )!;
29 |         }
30 |         catch (Exception e)
31 |         {
32 |             Console.WriteLine($"Could not parse environments file {environmentsFile.FullName}: {e.Message}");
33 |             Environment.Exit(1);
34 |         }
35 | 
36 |         if (rest)
37 |         {
38 |             KQLAnalyzerRESTService.LaunchRestServer(bindAddress, kqlEnvironments);
39 |             return;
40 |         }
41 | 
42 |         if (inputFile != null)
43 |         {
44 |             var analyzeRequest = new AnalyzeRequest();
45 |             try
46 |             {
47 |                 analyzeRequest = JsonSerializer.Deserialize<AnalyzeRequest>(
48 |                     File.ReadAllText(inputFile.FullName)
49 |                 )!;
50 |             }
51 |             catch (Exception e)
52 |             {
53 |                 Console.WriteLine($"Could not parse input file {inputFile.FullName}: {e.Message}");
54 |                 Environment.Exit(1);
55 |             }
56 | 
57 |             var environmentName = analyzeRequest.Environment;
58 |             if (!kqlEnvironments.TryGetValue(environmentName, out var environment))
59 |             {
60 |                 Console.WriteLine($"Could not find environment {environmentName}.");
61 |                 Environment.Exit(1);
62 |             }
63 | 
64 |             var results = KustoAnalyzer.AnalyzeQuery(
65 |                 analyzeRequest.Query,
66 |                 environment.ToGlobalState(),
67 |                 analyzeRequest.LocalData
68 |             );
69 |             Console.WriteLine(
70 |                 JsonSerializer.Serialize(
71 |                     results,
72 |                     new JsonSerializerOptions { WriteIndented = true }
73 |                 )
74 |             );
75 |             return;
76 |         }
77 | 
78 |         Console.WriteLine("Please provide either --input-file or --rest.");
79 |         Environment.Exit(1);
80 |     }
81 | }
82 | 


--------------------------------------------------------------------------------
/update_schemas.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | #
3 | python3 get_schemas.py > environments.json
4 | 


--------------------------------------------------------------------------------