├── .gitattributes
├── .github
└── workflows
│ ├── auto-deploy.yml
│ ├── codeql-analysis.yml
│ └── test-build.yml
├── .gitignore
├── CODE_OF_CONDUCT.md
├── Doc
├── CommandLineTool.md
├── PowerShellModule.md
├── PythonModule.md
└── Queries
│ ├── Linux
│ └── SyslogLogin.kql
│ └── Windows
│ ├── EtwDns.kql
│ ├── SimplifyEtwTcp.kql
│ └── SummarizeEtwTcp.kql
├── README.md
├── SECURITY.md
├── Source
├── Actions
│ ├── CreateReleaseAction
│ │ ├── action.yml
│ │ ├── build.cmd
│ │ ├── dist
│ │ │ ├── index.js
│ │ │ └── licenses.txt
│ │ ├── index.js
│ │ ├── package-lock.json
│ │ └── package.json
│ ├── SetupPythonDeploymentAction
│ │ ├── action.yml
│ │ ├── build.cmd
│ │ ├── dist
│ │ │ ├── index.js
│ │ │ └── licenses.txt
│ │ ├── index.js
│ │ ├── package-lock.json
│ │ └── package.json
│ └── SignAction
│ │ ├── action.yml
│ │ ├── build.cmd
│ │ ├── dist
│ │ ├── index.js
│ │ └── licenses.txt
│ │ ├── index.js
│ │ ├── package-lock.json
│ │ └── package.json
├── KqlPowerShell
│ ├── BaseCmdlet.cs
│ ├── KqlCmdlets.cs
│ ├── KqlPowerShell.csproj
│ ├── QueuedDictionaryOutput.cs
│ ├── README.md
│ └── RealTimeKql.psd1
├── KqlPython
│ ├── README.md
│ ├── __init__.py
│ ├── realtimekql.py
│ └── setup.py
├── KqlTools.sln
├── Microsoft.Syslog
│ ├── Internals
│ │ ├── BatchingQueue.cs
│ │ └── Observable.cs
│ ├── Microsoft.Syslog.csproj
│ ├── Model
│ │ ├── Enums.cs
│ │ ├── SyslogEntry.cs
│ │ └── SyslogExtensions.cs
│ ├── Parsing
│ │ ├── Extractors
│ │ │ ├── IValuesExtractor.cs
│ │ │ ├── IpAddressesExtractor.cs
│ │ │ └── KeywordValuesExtractorBase.cs
│ │ ├── ParserContext.cs
│ │ ├── ParserContextExtensions.cs
│ │ ├── Parsers
│ │ │ ├── ISyslogMessageParser.cs
│ │ │ ├── KeyValueListParser.cs
│ │ │ ├── PlainTextParser.cs
│ │ │ ├── Rfc3164SyslogParser.cs
│ │ │ ├── Rfc5424SyslogParser.cs
│ │ │ └── TimestampParseHelper.cs
│ │ ├── StringExtensions.cs
│ │ ├── SyslogChars.cs
│ │ └── SyslogParser.cs
│ ├── ReadMe.md
│ ├── SyslogClient.cs
│ ├── SyslogEventArgs.cs
│ ├── SyslogListener.cs
│ ├── SyslogSerializer.cs
│ └── UdpListener.cs
├── RealTimeKql
│ ├── CommandLineParsing
│ │ ├── Argument.cs
│ │ ├── CommandLineParser.cs
│ │ ├── Option.cs
│ │ └── Subcommand.cs
│ ├── Program.cs
│ ├── RealTimeKql.csproj
│ └── TestAssets
│ │ ├── SampleCsv.csv
│ │ ├── SampleEvtx.evtx
│ │ ├── SampleSyslog.txt
│ │ └── test.kql
├── RealTimeKqlLibrary
│ ├── CsvFileReader.cs
│ ├── EtlFileReader.cs
│ ├── EtwSession.cs
│ ├── EventComponent.cs
│ ├── EventProcessing
│ │ ├── CustomFunctions
│ │ │ ├── GetProcessName.cs
│ │ │ └── NetworkToHostPort.cs
│ │ └── EventProcessor.cs
│ ├── EvtxFileReader.cs
│ ├── Internals
│ │ ├── DictionaryDataReader.cs
│ │ ├── ModifierSubject.cs
│ │ ├── ObjectToDictionaryHelper.cs
│ │ └── Observable.cs
│ ├── Logging
│ │ ├── BaseLogger.cs
│ │ ├── ConsoleLogger.cs
│ │ └── WindowsLogger.cs
│ ├── Output
│ │ ├── AdxOutput.cs
│ │ ├── BlobOutput.cs
│ │ ├── ConsoleJsonOutput.cs
│ │ ├── ConsoleTableOutput.cs
│ │ ├── EventLogOutput.cs
│ │ ├── IOutput.cs
│ │ └── JsonFileOutput.cs
│ ├── RealTimeKqlLibrary.csproj
│ ├── SyslogEntryToDictionaryConverter.cs
│ ├── SyslogFileReader.cs
│ ├── SyslogKeywordValuesExtractor.cs
│ ├── SyslogPatternBasedValuesExtractor.cs
│ ├── SyslogServer.cs
│ └── WinlogRealTime.cs
└── RealTimeKqlTests
│ ├── Assets
│ ├── SampleCsv.csv
│ ├── SampleEvtx.evtx
│ ├── SampleSyslog.txt
│ ├── test.kql
│ └── test2.kql
│ ├── CommandLineParserTest.cs
│ └── RealTimeKqlTests.csproj
├── StandingQuery.jpg
└── license.txt
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Set the default behavior, in case people don't have core.autocrlf set.
2 | * text=auto
3 |
4 | # Use text conventions for commonly used text extensions.
5 | *.csv text
6 | *.ini text
7 | *.json text
8 | *.txt text
9 | *.xml text
10 |
11 | # Denote all files that are truly binary and should not be modified.
12 | *.dll binary
13 | *.exe binary
14 | *.gz binary
15 | *.ico binary
16 | *.jpg binary
17 | *.lib binary
18 | *.pdb binary
19 | *.pdf binary
20 | *.png binary
21 | *.wim binary
22 | *.zip binary
23 |
--------------------------------------------------------------------------------
/.github/workflows/auto-deploy.yml:
--------------------------------------------------------------------------------
1 | # Workflow to deploy a new command line tool release when a new tag is pushed
2 | name: Build and Release
3 |
4 | on:
5 | push:
6 | tags:
7 | - '*'
8 |
9 | jobs:
10 | setup:
11 | runs-on: windows-latest
12 |
13 | defaults:
14 | run:
15 | shell: powershell
16 |
17 | outputs:
18 | tag_name: ${{ steps.getnames.outputs.tag }}
19 | release_name: ${{ steps.getnames.outputs.release }}
20 |
21 | steps:
22 | - name: Checkout
23 | uses: actions/checkout@v2
24 |
25 | # Get tag name for new release
26 | - name: Get Tag and Release Names
27 | id: getnames
28 | run: |
29 | $tmp = '${{ github.ref }}'.split('/')
30 | $tag = $tmp[$tmp.length-1]
31 | $release = 'RealTimeKql ' + $tag
32 | echo "::set-output name=tag::$tag"
33 | echo "::set-output name=release::$release"
34 |
35 | commandlinetool:
36 | needs: setup
37 |
38 | runs-on: windows-latest
39 |
40 | defaults:
41 | run:
42 | shell: powershell
43 |
44 | steps:
45 | - name: Checkout
46 | uses: actions/checkout@v2
47 |
48 | # Run dotnet publish for all necessary binaries
49 | - name: Publish Binaries
50 | run: |
51 | dotnet clean Source/KqlTools.sln
52 | dotnet nuget locals all --clear
53 | dotnet publish Source/RealTimeKql/RealTimeKql.csproj -r win-x64 -f netcoreapp3.1 -c Release -p:PublishSingleFile=true -o ${{ runner.temp }}\win-x64
54 | dotnet publish Source/RealTimeKql/RealTimeKql.csproj -r linux-x64 -f netcoreapp3.1 -c Release -p:PublishSingleFile=true -o ${{ runner.temp }}\linux-x64
55 |
56 | # Compress release packages for win-x64
57 | - name: Compress Binaries Windows
58 | run: |
59 | mkdir ${{ github.workspace }}\ReleaseAssets
60 | copy Doc/Queries/Windows/* ${{ runner.temp }}\win-x64
61 | Compress-Archive -Path ${{ runner.temp }}\win-x64\* -DestinationPath "${{ github.workspace }}\ReleaseAssets\RealTimeKql.${{ env.TAG_NAME }}.zip"
62 |
63 | # Compress release packages for linux-x64
64 | - name: Compress Binaries Linux
65 | run: |
66 | copy Doc/Queries/Linux/* ${{ runner.temp }}\linux-x64
67 | cd ReleaseAssets
68 | tar -czvf "RealTimeKql.${{ env.TAG_NAME }}.tar.gz" ${{ runner.temp }}\linux-x64\*
69 |
70 | # Upload compressed binaries to latest release
71 | - name: Create Release Step
72 | uses: ./Source/Actions/CreateReleaseAction
73 | with:
74 | token: ${{ secrets.GITHUB_TOKEN }}
75 | tag_name: ${{ needs.setup.outputs.tag_name }}
76 | release_name: ${{ needs.setup.outputs.release_name }}
77 | directory: '${{ github.workspace }}\ReleaseAssets'
78 |
79 | pythonmodule:
80 | needs: setup
81 |
82 | runs-on: windows-latest
83 |
84 | defaults:
85 | run:
86 | shell: powershell
87 |
88 | env:
89 | Identity_Mapper: "namita-prakash:${{ secrets.NAPRAKAS_PYPI_KEY }};" # Add mapping from github username to pypi api key secret
90 |
91 | steps:
92 | - name: Checkout
93 | uses: actions/checkout@v2
94 |
95 | # Get PyPi API key for current user
96 | - name: Set API key
97 | run: |
98 | $ids = $env:Identity_Mapper -split ";"
99 | $mapper = New-Object System.Collections.Generic.Dictionary"[String,String]"
100 | foreach ($id in $ids) { $pair = $id -split ":"; $mapper.Add($pair[0],$pair[1]) }
101 | $key = $mapper["${{ github.actor }}"]
102 | echo "PYPI_API_KEY=$key" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
103 |
104 | # Run dotnet publish for all necessary binaries
105 | - name: Generate published dependencies
106 | run: |
107 | dotnet clean Source/KqlTools.sln
108 | dotnet nuget locals all --clear
109 | dotnet publish Source/RealTimeKqlLibrary/RealTimeKqlLibrary.csproj -r win-x64 -f net472 -c Release -o ${{ runner.temp }}\python\realtimekql\lib
110 |
111 | # Set up python build directory
112 | - name: Set up python build directory step
113 | run: |
114 | copy Source/KqlPython/* ${{ runner.temp }}\python\realtimekql
115 | cd ${{ runner.temp }}\python\realtimekql
116 | "${{ needs.setup.outputs.tag_name }}" | Out-File -FilePath VERSION.txt -Encoding ASCII -NoNewline
117 | 'directory = r"${{ runner.temp }}\python\realtimekql"' | Out-File -FilePath kqlpythondir.py -Encoding ASCII -NoNewline
118 |
119 | # Build python wheel
120 | - name: Build Python Wheel Step
121 | run: |
122 | cd ${{ runner.temp }}\python\realtimekql
123 | python -m pip install -U pip wheel setuptools build
124 | python -m build
125 |
126 | # Deploy python module
127 | - name: Deploy Python Module
128 | run: |
129 | cd ${{ runner.temp }}\python\realtimekql\
130 | python -m pip install --user --upgrade twine
131 | python -m twine upload dist\* -u __token__ -p $env:PYPI_API_KEY
132 |
133 | powershellmodule:
134 | needs: setup
135 |
136 | runs-on: windows-latest
137 |
138 | defaults:
139 | run:
140 | shell: powershell
141 |
142 | env:
143 | Identity_Mapper: "namita-prakash:${{ secrets.NAPRAKAS_POWERSHELL_API_KEY }};" # Add mapping from github username to powershell gallery api key secret
144 |
145 | steps:
146 | - name: Checkout
147 | uses: actions/checkout@v2
148 |
149 | # Get PyPi API key for current user
150 | - name: Set API key
151 | run: |
152 | $ids = $env:Identity_Mapper -split ";"
153 | $mapper = New-Object System.Collections.Generic.Dictionary"[String,String]"
154 | foreach ($id in $ids) { $pair = $id -split ":"; $mapper.Add($pair[0],$pair[1]) }
155 | $key = $mapper["${{ github.actor }}"]
156 | echo "POWERSHELL_API_KEY=$key" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
157 |
158 | # Run dotnet publish for all necessary binaries
159 | - name: Generate published dependencies
160 | run: |
161 | dotnet clean Source/KqlTools.sln
162 | dotnet nuget locals all --clear
163 | dotnet publish Source/KqlPowerShell/KqlPowerShell.csproj -c Release -o ${{ runner.temp }}\powershell\RealTimeKql
164 |
165 | # Generate module manifest & publish module
166 | - name: Generate module manifest & publish module
167 | run: |
168 | copy Source/KqlPowerShell/RealTimeKql.psd1 ${{ runner.temp }}\powershell\RealTimeKql
169 | cd ${{ runner.temp }}\powershell\RealTimeKql
170 | Update-ModuleManifest RealTimeKql.psd1 -ModuleVersion ${{ needs.setup.outputs.tag_name }}
171 | Test-ModuleManifest RealTimeKql.psd1
172 | Publish-Module -Path ${{ runner.temp }}\powershell\RealTimeKql -NuGetApiKey $env:POWERSHELL_API_KEY
173 |
--------------------------------------------------------------------------------
/.github/workflows/codeql-analysis.yml:
--------------------------------------------------------------------------------
1 | # For most projects, this workflow file will not need changing; you simply need
2 | # to commit it to your repository.
3 | #
4 | # You may wish to alter this file to override the set of languages analyzed,
5 | # or to provide custom queries or build logic.
6 | #
7 | # ******** NOTE ********
8 | # We have attempted to detect the languages in your repository. Please check
9 | # the `language` matrix defined below to confirm you have the correct set of
10 | # supported CodeQL languages.
11 | #
12 | name: "CodeQL"
13 |
14 | on:
15 | push:
16 | branches: [ master ]
17 | pull_request:
18 | # The branches below must be a subset of the branches above
19 | branches: [ master ]
20 | schedule:
21 | - cron: '29 18 * * 5'
22 |
23 | jobs:
24 | analyze:
25 | name: Analyze
26 | runs-on: ubuntu-latest
27 |
28 | strategy:
29 | fail-fast: false
30 | matrix:
31 | language: [ 'csharp' ]
32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
33 | # Learn more:
34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
35 |
36 | steps:
37 | - name: Checkout repository
38 | uses: actions/checkout@v2
39 |
40 | # Initializes the CodeQL tools for scanning.
41 | - name: Initialize CodeQL
42 | uses: github/codeql-action/init@v1
43 | with:
44 | languages: ${{ matrix.language }}
45 | # If you wish to specify custom queries, you can do so here or in a config file.
46 | # By default, queries listed here will override any specified in a config file.
47 | # Prefix the list here with "+" to use these queries and those in the config file.
48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main
49 |
50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
51 | # If this step fails, then you should remove it and run the build manually (see below)
52 | - name: Autobuild
53 | uses: github/codeql-action/autobuild@v1
54 |
55 | # ℹ️ Command-line programs to run using the OS shell.
56 | # 📚 https://git.io/JvXDl
57 |
58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
59 | # and modify them (or add more) to build your code if your project
60 | # uses a compiled language
61 |
62 | #- run: |
63 | # make bootstrap
64 | # make release
65 |
66 | - name: Perform CodeQL Analysis
67 | uses: github/codeql-action/analyze@v1
68 |
--------------------------------------------------------------------------------
/.github/workflows/test-build.yml:
--------------------------------------------------------------------------------
1 | # This workflow will build a project and execute all unit tests in its solution
2 |
3 | name: Test Build
4 |
5 | on:
6 | push:
7 | branches: [ master ]
8 | pull_request:
9 | branches: [ master ]
10 |
11 | jobs:
12 |
13 | build:
14 |
15 | runs-on: windows-latest
16 |
17 | env:
18 | Solution_Path: Source\KqlTools.sln # Path to solution
19 | Project_Path: Source\RealTimeKql\RealTimeKql.csproj # Path to project
20 |
21 | steps:
22 | - name: Checkout
23 | uses: actions/checkout@v2
24 |
25 | # Clean solution
26 | - name: Clean solution
27 | run: |
28 | dotnet clean $env:Solution_Path
29 | dotnet nuget locals all --clear
30 |
31 | # Build project
32 | - name: Build project
33 | run: dotnet build $env:Project_Path
34 |
35 | # Execute all unit tests in solution
36 | - name: Execute unit tests
37 | run: dotnet test $env:Solution_Path
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Microsoft Open Source Code of Conduct
2 |
3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
4 |
5 | Resources:
6 |
7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/)
8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/)
9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns
10 |
--------------------------------------------------------------------------------
/Doc/CommandLineTool.md:
--------------------------------------------------------------------------------
1 | # Real-Time KQL Command Line Tool
2 |
3 | A command line tool to explore and process real-time streams of events.
4 |
5 |
6 |
7 | ## Contents
8 |
9 | * [Download & Setup](#Setup)
10 | * [Usage](#Usage)
11 | * [Tracing ETW Tcp Events](#Etw)
12 | * [Tracing Local Syslog Events](#Syslog)
13 |
14 |
15 |
16 | ## Download & Setup
17 |
18 | Download the latest release from the [Releases](https://github.com/microsoft/KqlTools/releases/) page. For Windows, download RealTimeKql.zip. For Linux, download RealTimeKql.tar.gz. The zip files include an executable for Real-Time KQL as well as some sample queries.
19 |
20 |
21 |
22 | ## Usage
23 |
24 | ```
25 | Usage: RealTimeKql [] [--options] [[