├── .gitattributes ├── .github └── workflows │ ├── auto-deploy.yml │ ├── codeql-analysis.yml │ └── test-build.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── Doc ├── CommandLineTool.md ├── PowerShellModule.md ├── PythonModule.md └── Queries │ ├── Linux │ └── SyslogLogin.kql │ └── Windows │ ├── EtwDns.kql │ ├── SimplifyEtwTcp.kql │ └── SummarizeEtwTcp.kql ├── README.md ├── SECURITY.md ├── Source ├── Actions │ ├── CreateReleaseAction │ │ ├── action.yml │ │ ├── build.cmd │ │ ├── dist │ │ │ ├── index.js │ │ │ └── licenses.txt │ │ ├── index.js │ │ ├── package-lock.json │ │ └── package.json │ ├── SetupPythonDeploymentAction │ │ ├── action.yml │ │ ├── build.cmd │ │ ├── dist │ │ │ ├── index.js │ │ │ └── licenses.txt │ │ ├── index.js │ │ ├── package-lock.json │ │ └── package.json │ └── SignAction │ │ ├── action.yml │ │ ├── build.cmd │ │ ├── dist │ │ ├── index.js │ │ └── licenses.txt │ │ ├── index.js │ │ ├── package-lock.json │ │ └── package.json ├── KqlPowerShell │ ├── BaseCmdlet.cs │ ├── KqlCmdlets.cs │ ├── KqlPowerShell.csproj │ ├── QueuedDictionaryOutput.cs │ ├── README.md │ └── RealTimeKql.psd1 ├── KqlPython │ ├── README.md │ ├── __init__.py │ ├── realtimekql.py │ └── setup.py ├── KqlTools.sln ├── Microsoft.Syslog │ ├── Internals │ │ ├── BatchingQueue.cs │ │ └── Observable.cs │ ├── Microsoft.Syslog.csproj │ ├── Model │ │ ├── Enums.cs │ │ ├── SyslogEntry.cs │ │ └── SyslogExtensions.cs │ ├── Parsing │ │ ├── Extractors │ │ │ ├── IValuesExtractor.cs │ │ │ ├── IpAddressesExtractor.cs │ │ │ └── KeywordValuesExtractorBase.cs │ │ ├── ParserContext.cs │ │ ├── ParserContextExtensions.cs │ │ ├── Parsers │ │ │ ├── ISyslogMessageParser.cs │ │ │ ├── KeyValueListParser.cs │ │ │ ├── PlainTextParser.cs │ │ │ ├── Rfc3164SyslogParser.cs │ │ │ ├── Rfc5424SyslogParser.cs │ │ │ └── TimestampParseHelper.cs │ │ ├── StringExtensions.cs │ │ ├── SyslogChars.cs │ │ └── SyslogParser.cs │ ├── ReadMe.md │ ├── SyslogClient.cs │ ├── SyslogEventArgs.cs │ ├── SyslogListener.cs │ ├── SyslogSerializer.cs │ └── UdpListener.cs ├── RealTimeKql │ ├── CommandLineParsing │ │ ├── Argument.cs │ │ ├── CommandLineParser.cs │ │ ├── Option.cs │ │ └── Subcommand.cs │ ├── Program.cs │ ├── RealTimeKql.csproj │ └── TestAssets │ │ ├── SampleCsv.csv │ │ ├── SampleEvtx.evtx │ │ ├── SampleSyslog.txt │ │ └── test.kql ├── RealTimeKqlLibrary │ ├── CsvFileReader.cs │ ├── EtlFileReader.cs │ ├── EtwSession.cs │ ├── EventComponent.cs │ ├── EventProcessing │ │ ├── CustomFunctions │ │ │ ├── GetProcessName.cs │ │ │ └── NetworkToHostPort.cs │ │ └── EventProcessor.cs │ ├── EvtxFileReader.cs │ ├── Internals │ │ ├── DictionaryDataReader.cs │ │ ├── ModifierSubject.cs │ │ ├── ObjectToDictionaryHelper.cs │ │ └── Observable.cs │ ├── Logging │ │ ├── BaseLogger.cs │ │ ├── ConsoleLogger.cs │ │ └── WindowsLogger.cs │ ├── Output │ │ ├── AdxOutput.cs │ │ ├── BlobOutput.cs │ │ ├── ConsoleJsonOutput.cs │ │ ├── ConsoleTableOutput.cs │ │ ├── EventLogOutput.cs │ │ ├── IOutput.cs │ │ └── JsonFileOutput.cs │ ├── RealTimeKqlLibrary.csproj │ ├── SyslogEntryToDictionaryConverter.cs │ ├── SyslogFileReader.cs │ ├── SyslogKeywordValuesExtractor.cs │ ├── SyslogPatternBasedValuesExtractor.cs │ ├── SyslogServer.cs │ └── WinlogRealTime.cs └── RealTimeKqlTests │ ├── Assets │ ├── SampleCsv.csv │ ├── SampleEvtx.evtx │ ├── SampleSyslog.txt │ ├── test.kql │ └── test2.kql │ ├── CommandLineParserTest.cs │ └── RealTimeKqlTests.csproj ├── StandingQuery.jpg └── license.txt /.gitattributes: -------------------------------------------------------------------------------- 1 | # Set the default behavior, in case people don't have core.autocrlf set. 2 | * text=auto 3 | 4 | # Use text conventions for commonly used text extensions. 5 | *.csv text 6 | *.ini text 7 | *.json text 8 | *.txt text 9 | *.xml text 10 | 11 | # Denote all files that are truly binary and should not be modified. 12 | *.dll binary 13 | *.exe binary 14 | *.gz binary 15 | *.ico binary 16 | *.jpg binary 17 | *.lib binary 18 | *.pdb binary 19 | *.pdf binary 20 | *.png binary 21 | *.wim binary 22 | *.zip binary 23 | -------------------------------------------------------------------------------- /.github/workflows/auto-deploy.yml: -------------------------------------------------------------------------------- 1 | # Workflow to deploy a new command line tool release when a new tag is pushed 2 | name: Build and Release 3 | 4 | on: 5 | push: 6 | tags: 7 | - '*' 8 | 9 | jobs: 10 | setup: 11 | runs-on: windows-latest 12 | 13 | defaults: 14 | run: 15 | shell: powershell 16 | 17 | outputs: 18 | tag_name: ${{ steps.getnames.outputs.tag }} 19 | release_name: ${{ steps.getnames.outputs.release }} 20 | 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v2 24 | 25 | # Get tag name for new release 26 | - name: Get Tag and Release Names 27 | id: getnames 28 | run: | 29 | $tmp = '${{ github.ref }}'.split('/') 30 | $tag = $tmp[$tmp.length-1] 31 | $release = 'RealTimeKql ' + $tag 32 | echo "::set-output name=tag::$tag" 33 | echo "::set-output name=release::$release" 34 | 35 | commandlinetool: 36 | needs: setup 37 | 38 | runs-on: windows-latest 39 | 40 | defaults: 41 | run: 42 | shell: powershell 43 | 44 | steps: 45 | - name: Checkout 46 | uses: actions/checkout@v2 47 | 48 | # Run dotnet publish for all necessary binaries 49 | - name: Publish Binaries 50 | run: | 51 | dotnet clean Source/KqlTools.sln 52 | dotnet nuget locals all --clear 53 | dotnet publish Source/RealTimeKql/RealTimeKql.csproj -r win-x64 -f netcoreapp3.1 -c Release -p:PublishSingleFile=true -o ${{ runner.temp }}\win-x64 54 | dotnet publish Source/RealTimeKql/RealTimeKql.csproj -r linux-x64 -f netcoreapp3.1 -c Release -p:PublishSingleFile=true -o ${{ runner.temp }}\linux-x64 55 | 56 | # Compress release packages for win-x64 57 | - name: Compress Binaries Windows 58 | run: | 59 | mkdir ${{ github.workspace }}\ReleaseAssets 60 | copy Doc/Queries/Windows/* ${{ runner.temp }}\win-x64 61 | Compress-Archive -Path ${{ runner.temp }}\win-x64\* -DestinationPath "${{ github.workspace }}\ReleaseAssets\RealTimeKql.${{ env.TAG_NAME }}.zip" 62 | 63 | # Compress release packages for linux-x64 64 | - name: Compress Binaries Linux 65 | run: | 66 | copy Doc/Queries/Linux/* ${{ runner.temp }}\linux-x64 67 | cd ReleaseAssets 68 | tar -czvf "RealTimeKql.${{ env.TAG_NAME }}.tar.gz" ${{ runner.temp }}\linux-x64\* 69 | 70 | # Upload compressed binaries to latest release 71 | - name: Create Release Step 72 | uses: ./Source/Actions/CreateReleaseAction 73 | with: 74 | token: ${{ secrets.GITHUB_TOKEN }} 75 | tag_name: ${{ needs.setup.outputs.tag_name }} 76 | release_name: ${{ needs.setup.outputs.release_name }} 77 | directory: '${{ github.workspace }}\ReleaseAssets' 78 | 79 | pythonmodule: 80 | needs: setup 81 | 82 | runs-on: windows-latest 83 | 84 | defaults: 85 | run: 86 | shell: powershell 87 | 88 | env: 89 | Identity_Mapper: "namita-prakash:${{ secrets.NAPRAKAS_PYPI_KEY }};" # Add mapping from github username to pypi api key secret 90 | 91 | steps: 92 | - name: Checkout 93 | uses: actions/checkout@v2 94 | 95 | # Get PyPi API key for current user 96 | - name: Set API key 97 | run: | 98 | $ids = $env:Identity_Mapper -split ";" 99 | $mapper = New-Object System.Collections.Generic.Dictionary"[String,String]" 100 | foreach ($id in $ids) { $pair = $id -split ":"; $mapper.Add($pair[0],$pair[1]) } 101 | $key = $mapper["${{ github.actor }}"] 102 | echo "PYPI_API_KEY=$key" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append 103 | 104 | # Run dotnet publish for all necessary binaries 105 | - name: Generate published dependencies 106 | run: | 107 | dotnet clean Source/KqlTools.sln 108 | dotnet nuget locals all --clear 109 | dotnet publish Source/RealTimeKqlLibrary/RealTimeKqlLibrary.csproj -r win-x64 -f net472 -c Release -o ${{ runner.temp }}\python\realtimekql\lib 110 | 111 | # Set up python build directory 112 | - name: Set up python build directory step 113 | run: | 114 | copy Source/KqlPython/* ${{ runner.temp }}\python\realtimekql 115 | cd ${{ runner.temp }}\python\realtimekql 116 | "${{ needs.setup.outputs.tag_name }}" | Out-File -FilePath VERSION.txt -Encoding ASCII -NoNewline 117 | 'directory = r"${{ runner.temp }}\python\realtimekql"' | Out-File -FilePath kqlpythondir.py -Encoding ASCII -NoNewline 118 | 119 | # Build python wheel 120 | - name: Build Python Wheel Step 121 | run: | 122 | cd ${{ runner.temp }}\python\realtimekql 123 | python -m pip install -U pip wheel setuptools build 124 | python -m build 125 | 126 | # Deploy python module 127 | - name: Deploy Python Module 128 | run: | 129 | cd ${{ runner.temp }}\python\realtimekql\ 130 | python -m pip install --user --upgrade twine 131 | python -m twine upload dist\* -u __token__ -p $env:PYPI_API_KEY 132 | 133 | powershellmodule: 134 | needs: setup 135 | 136 | runs-on: windows-latest 137 | 138 | defaults: 139 | run: 140 | shell: powershell 141 | 142 | env: 143 | Identity_Mapper: "namita-prakash:${{ secrets.NAPRAKAS_POWERSHELL_API_KEY }};" # Add mapping from github username to powershell gallery api key secret 144 | 145 | steps: 146 | - name: Checkout 147 | uses: actions/checkout@v2 148 | 149 | # Get PyPi API key for current user 150 | - name: Set API key 151 | run: | 152 | $ids = $env:Identity_Mapper -split ";" 153 | $mapper = New-Object System.Collections.Generic.Dictionary"[String,String]" 154 | foreach ($id in $ids) { $pair = $id -split ":"; $mapper.Add($pair[0],$pair[1]) } 155 | $key = $mapper["${{ github.actor }}"] 156 | echo "POWERSHELL_API_KEY=$key" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append 157 | 158 | # Run dotnet publish for all necessary binaries 159 | - name: Generate published dependencies 160 | run: | 161 | dotnet clean Source/KqlTools.sln 162 | dotnet nuget locals all --clear 163 | dotnet publish Source/KqlPowerShell/KqlPowerShell.csproj -c Release -o ${{ runner.temp }}\powershell\RealTimeKql 164 | 165 | # Generate module manifest & publish module 166 | - name: Generate module manifest & publish module 167 | run: | 168 | copy Source/KqlPowerShell/RealTimeKql.psd1 ${{ runner.temp }}\powershell\RealTimeKql 169 | cd ${{ runner.temp }}\powershell\RealTimeKql 170 | Update-ModuleManifest RealTimeKql.psd1 -ModuleVersion ${{ needs.setup.outputs.tag_name }} 171 | Test-ModuleManifest RealTimeKql.psd1 172 | Publish-Module -Path ${{ runner.temp }}\powershell\RealTimeKql -NuGetApiKey $env:POWERSHELL_API_KEY 173 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '29 18 * * 5' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: [ 'csharp' ] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v2 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v1 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v1 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@v1 68 | -------------------------------------------------------------------------------- /.github/workflows/test-build.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a project and execute all unit tests in its solution 2 | 3 | name: Test Build 4 | 5 | on: 6 | push: 7 | branches: [ master ] 8 | pull_request: 9 | branches: [ master ] 10 | 11 | jobs: 12 | 13 | build: 14 | 15 | runs-on: windows-latest 16 | 17 | env: 18 | Solution_Path: Source\KqlTools.sln # Path to solution 19 | Project_Path: Source\RealTimeKql\RealTimeKql.csproj # Path to project 20 | 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v2 24 | 25 | # Clean solution 26 | - name: Clean solution 27 | run: | 28 | dotnet clean $env:Solution_Path 29 | dotnet nuget locals all --clear 30 | 31 | # Build project 32 | - name: Build project 33 | run: dotnet build $env:Project_Path 34 | 35 | # Execute all unit tests in solution 36 | - name: Execute unit tests 37 | run: dotnet test $env:Solution_Path -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /Doc/CommandLineTool.md: -------------------------------------------------------------------------------- 1 | # Real-Time KQL Command Line Tool 2 | 3 | A command line tool to explore and process real-time streams of events. 4 | 5 | 6 | 7 | ## Contents 8 | 9 | * [Download & Setup](#Setup) 10 | * [Usage](#Usage) 11 | * [Tracing ETW Tcp Events](#Etw) 12 | * [Tracing Local Syslog Events](#Syslog) 13 | 14 | 15 | 16 | ## Download & Setup 17 | 18 | Download the latest release from the [Releases](https://github.com/microsoft/KqlTools/releases/) page. For Windows, download RealTimeKql.zip. For Linux, download RealTimeKql.tar.gz. The zip files include an executable for Real-Time KQL as well as some sample queries. 19 | 20 | 21 | 22 | ## Usage 23 | 24 | ``` 25 | Usage: RealTimeKql [] [--options] [[] []] [--query=] 26 | 27 | input commands 28 | etw Listen to real-time ETW session. See Event Trace Sessions in Perfmon 29 | etl Process the past event in Event Trace File (.etl) recorded via ETW 30 | winlog Listen for new events in a Windows OS log. See Windows Logs in Eventvwr 31 | evtx Process the past events recorded in Windows log file on disk 32 | csv Process past events recorded in Comma Separated File 33 | syslog Process real-time syslog messages written to local log file 34 | syslogserver [options] Listen to syslog messages on a UDP port 35 | 36 | output commands 37 | json [file.json] Optional and default. Events printed to console in JSON format. If filename is specified immediately after, events will be written to the file in JSON format. 38 | table Optional, events printed to console in table format 39 | adx Ingest output to Azure Data Explorer 40 | blob Ingest output to Azure Blob Storage 41 | 42 | query file 43 | -q|--query Optional, apply this KQL query to the input stream. If omitted, the stream is propagated without processing to the output 44 | 45 | Use "RealTimeKql [command] -h|--help" for more information about a command. 46 | ``` 47 | 48 | 49 | 50 | ## Tracing ETW TCP Events 51 | 52 | This walkthrough demonstrates how to use RealTimeKql to explore TCP events emitted by an Event Tracing for Windows (ETW) provider. 53 | 54 | ### Start an ETW Trace: 55 | 56 | logman is a utility that allows you to start an Event Trace Session for a specific ETW provider or set of providers. In an Administrator command prompt, run this command to start an event trace session for the Etw Tcp provider: 57 | 58 | ``` 59 | logman.exe create trace tcp -rt -nb 2 2 -bs 1024 -p {7dd42a49-5329-4832-8dfd-43d979153a88} 0xffffffffffffffff -ets 60 | ``` 61 | 62 | 63 | 64 | ### Look at raw TCP data: 65 | 66 | From within an Administrator command prompt, navigate to the folder where your RealTimeKql.exe is stored and run the following command: 67 | 68 | ``` 69 | RealTimeKql etw tcp json 70 | ``` 71 | 72 | 73 | 74 | ### Simplify TCP event output: 75 | 76 | You can use one of the sample queries provided to simplify the TCP data: 77 | 78 | ``` 79 | RealTimeKql etw tcp table --query=SimplifyEtwTcp.kql 80 | ``` 81 | 82 | 83 | 84 | ### Aggregate TCP events into 30-second intervals: 85 | 86 | To reduce the volume of events printing to your screen, you can apply a different query that aggregates events into 30-second windows: 87 | 88 | ``` 89 | RealTimeKql etw tcp table --query=SummarizeEtwTcp.kql 90 | ``` 91 | 92 | 93 | 94 | ### Stop an ETW Trace: 95 | 96 | Once you are done exploring events, you can stop the Event Trace Session you started with logman by running the following command: 97 | 98 | ``` 99 | logman.exe stop tcp -ets 100 | ``` 101 | 102 | 103 | 104 | ## Tracing Local Syslog Events 105 | 106 | This walkthrough demonstrates how to use RealTimeKql to explore the local syslog authentication log on a Linux machine. (This log is usually stored as **/var/log/auth.log**) 107 | 108 | ### Experiment Set-Up: 109 | 110 | In one terminal window (Terminal A), navigate to the folder where the Kql Tools are stored. In a second terminal window (Terminal B), prepare to login to your machine via ssh. 111 | 112 | 113 | 114 | ### Look at raw syslog events: 115 | 116 | In terminal A, run: 117 | 118 | ``` 119 | tail -f /var/log/auth.log 120 | ``` 121 | 122 | While tail is running in terminal A, use terminal B to try logging into your machine. You can also use terminal B to intentionally fail logging into your machine to see what events are generated on a failed login event. 123 | 124 | 125 | 126 | ### Look at syslog events with Real-Time KQL: 127 | 128 | In terminal A, run: 129 | 130 | ``` 131 | sudo ./RealTimeKql syslog /var/log/auth.log json 132 | ``` 133 | 134 | While RealTimeKql is running in terminal A, use terminal B to try logging in, both successfully and unsuccessfully, to your machine. 135 | 136 | 137 | 138 | ### Simplify local syslog event output: 139 | 140 | You can use one of the sample queries provided to simplify Syslog data. In terminal A, run: 141 | 142 | ``` 143 | sudo ./RealTimeKql syslog /var/log/auth.log table --query=SyslogLogin.kql 144 | ``` 145 | 146 | In terminal B, try logging in and out, successfully and unsuccessfully to see some output. 147 | -------------------------------------------------------------------------------- /Doc/PowerShellModule.md: -------------------------------------------------------------------------------- 1 | # Real-Time KQL for PowerShell 2 | 3 | ## Contents 4 | 5 | * [Download & Setup](#Setup) 6 | * [Usage](#Usage) 7 | * [Tracing ETW TCP Events](#Etw) 8 | 9 | ## Download & Setup 10 | 11 | In an Administrator PowerShell Window, run: 12 | 13 | ``` 14 | Install-Module -Name RealTimeKql 15 | ``` 16 | 17 | 18 | 19 | ## Usage 20 | 21 | To see all available cmdlets, run: 22 | 23 | ``` 24 | Get-Command -Module RealTimeKql 25 | ``` 26 | 27 | To use the cmdlets in the module, run: 28 | 29 | ``` 30 | Import-Module RealTimeKql 31 | ``` 32 | 33 | 34 | 35 | ## Tracing ETW TCP Events 36 | 37 | This walkthrough demonstrates how to use the Real-Time KQL PowerShell Module to explore TCP events emitted by an Event Tracing for Windows (ETW) provider. This walkthrough will use an **Administrator PowerShell** window throughout. 38 | 39 | ### Start an ETW Trace: 40 | 41 | logman is a utility that allows you to start an Event Trace Session for a specific ETW provider or set of providers. Run this command to start an event trace session for the Etw TCP provider: 42 | 43 | ``` 44 | logman.exe create trace tcp -rt -nb 2 2 -bs 1024 -p 'Microsoft-Windows-Kernel-Network' 0xffffffffffffffff -ets 45 | ``` 46 | 47 | By running `create trace tcp`, this session has been named "tcp". 48 | 49 | ### Import RealTimeKql: 50 | 51 | ``` 52 | Import-Module RealTimeKql 53 | ``` 54 | 55 | ### Look at raw TCP data: 56 | 57 | Run the Get-EtwSession cmdlet to see Etw data. Here "tcp" is the session name used earlier in the logman command to create the event trace session: 58 | 59 | ``` 60 | Get-EtwSession tcp 61 | ``` 62 | 63 | ### Simplify TCP event output: 64 | 65 | Store this query in a .kql file (for this exercise, we'll assume the file is named **query.kql**): 66 | 67 | ``` 68 | EtwTcp 69 | | where Provider == "Microsoft-Windows-Kernel-Network" 70 | | where EventId in (10, 11) 71 | | extend ProcessName = getprocessname(EventData.PID) 72 | | extend Source = strcat(EventData.saddr, ":", ntohs(EventData.sport)) 73 | | extend Destination = strcat(EventData.daddr, ":", ntohs(EventData.dport)) 74 | | project Provider, Source, Destination, Opcode, ProcessName 75 | ``` 76 | 77 | You can now use this query to simplify the output you see: 78 | 79 | ``` 80 | Get-EtwSession tcp -Query | Format-Table 81 | ``` 82 | 83 | ### Aggregate TCP events into 30-second intervals: 84 | 85 | To reduce the volume of events printing to your screen, you can apply a different query that aggregates events into 30-second windows. Store this query in a .kql file (for this exercise, we'll assume the file is named **summarize.kql**): 86 | 87 | ``` 88 | EtwTcp 89 | | where EventId in (10, 11) 90 | | extend ProcessName = getprocessname(EventData.PID) 91 | | extend Source = EventData.saddr, SourcePort = ntohs(EventData.sport) 92 | | extend Destination = EventData.daddr, DestinationPort = ntohs(EventData.dport) 93 | | extend Size = EventData.size 94 | | extend ProcessId = EventData.PID 95 | | summarize Count = count(), Bytes = sum(Size) by bin(TimeCreated, 30s), SourceIpAddress, SourcePort, DestinationIpAddress, DestinationPort, EventId, ProcessId, ProcessName 96 | ``` 97 | 98 | Pass in this new query to the Get-EtwSession cmdlet: 99 | 100 | ``` 101 | Get-EtwSession tcp -Query | Format-Table 102 | ``` 103 | 104 | ### Stop an ETW Trace: 105 | 106 | Once you are done exploring events, you can stop the Event Trace Session you started with logman by running the following command: 107 | 108 | ``` 109 | logman.exe stop tcp -ets 110 | ``` 111 | 112 | -------------------------------------------------------------------------------- /Doc/PythonModule.md: -------------------------------------------------------------------------------- 1 | # Real-Time KQL for Python 2 | 3 | ## Contents 4 | 5 | * [Download & Setup](#Setup) 6 | * [Usage](#Usage) 7 | 8 | ## Download & Setup 9 | 10 | In an Administrator Command Prompt, Anaconda Prompt, or any elevated terminal window of your choosing, run: 11 | 12 | ``` 13 | pip install realtimekql 14 | ``` 15 | 16 | Using a virtual environment of some sort is not required, but is recommended. 17 | 18 | ## Usage 19 | 20 | Real-Time KQL is broken up into three parts: the output, the query, and the input. 21 | 22 | ### The Output 23 | 24 | Real-Time KQL for Python has a `PythonOutput` class that allows you to customize what happens to events when they are outputted. The simplest usage of the `PythonOutput` class is to instantiate it with no parameters. This will print events to console in JSON format: 25 | 26 | ``` 27 | >>> from realtimekql import * 28 | >>> o = PythonOutput() 29 | ``` 30 | 31 | To customize the output, you can pass in any Python function that takes a dictionary as the only parameter to the `PythonOutput` class. For example, this function stores events in a list to use them later: 32 | 33 | ``` 34 | >>> events = [] 35 | >>> def storeEvents(event): 36 | ... events.append(event) 37 | ... 38 | >>> from realtimekql import * 39 | >>> o = PythonOutput(storeEvents) 40 | ``` 41 | 42 | The `PythonAdxOutput` class allows you to ingest data to an Azure Data Explorer (Kusto) table through queued ingestion. The class can be instantiated as follows: 43 | 44 | ``` 45 | >>> from realtimekql import * 46 | >>> o = PythonAdxOutput("YourCluster.kusto.windows.net", "YourDatabase", "YourTable", "YourClientId", "YourClientSecret", "YourAuthorityId", resetTable=True) 47 | ``` 48 | 49 | 50 | 51 | ### The Query 52 | 53 | You can optionally pass a .kql query into Real-Time KQL to filter, transform, and enrich your events before they even reach the output stage. 54 | 55 | 56 | 57 | ### The Input 58 | 59 | Real-Time KQL supports various real-time and file input sources. Each input class takes a unique set of arguments, an instance of one of the output classes, as well as an optional path to a query file. This prints real-time Etw TCP events to console in JSON format: 60 | 61 | ``` 62 | >>> from realtimekql import * 63 | >>> o = PythonOutput() 64 | >>> e = EtwSession("tcp", o) 65 | >>> e.Start() 66 | ``` 67 | 68 | Here are all the supported input options and how to use them: 69 | 70 | ``` 71 | EtwSession(sessionName, o, q) 72 | EtlFileReader(filePath, o, q) 73 | WinlogRealTime(logName, o, q) 74 | EvtxFileReader(filePath, o, q) 75 | CsvFileReader(filePath, o, q) 76 | ``` 77 | 78 | The variables `o` and `q` represent the output part and the query part respectively. The query part is optional and can be left out. 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /Doc/Queries/Linux/SyslogLogin.kql: -------------------------------------------------------------------------------- 1 | Login 2 | | extend Activity = extract("((Failed|Accepted)\\s\\w*)\\sfor\\s(\\w*)", 1, Payload) 3 | | where isnotnull(Activity) and isnotempty(Activity) 4 | | extend User = extract("((Failed|Accepted)\\s\\w*)\\sfor\\s(\\w*)", 3, Payload) 5 | | where isnotnull(Activity) and isnotempty(Activity) 6 | | project User, Activity 7 | -------------------------------------------------------------------------------- /Doc/Queries/Windows/EtwDns.kql: -------------------------------------------------------------------------------- 1 | EtwDns 2 | | extend QueryResults = extract("(\\d+\\.\\d+\\.\\d+\\.\\d+)", 1, EventData.QueryResults) 3 | | where isnotnull(QueryResults) and isnotempty(QueryResults) 4 | | where isnotnull(EventData.QueryName) and isnotempty(EventData.QueryName) 5 | | extend QueryName = EventData.QueryName 6 | | project TimeCreated, QueryResults, QueryName -------------------------------------------------------------------------------- /Doc/Queries/Windows/SimplifyEtwTcp.kql: -------------------------------------------------------------------------------- 1 | EtwTcp 2 | | where EventId in (10, 11) 3 | | extend ProcessName = getprocessname(EventData.PID) 4 | | extend SourceIpAddress = strcat(EventData.saddr, ":", ntohs(EventData.sport)) 5 | | extend DestinationIpAddress = strcat(EventData.daddr, ":", ntohs(EventData.dport)) 6 | | project SourceIpAddress, DestinationIpAddress, Opcode, ProcessName -------------------------------------------------------------------------------- /Doc/Queries/Windows/SummarizeEtwTcp.kql: -------------------------------------------------------------------------------- 1 | EtwTcp 2 | | where Provider == 'Microsoft-Windows-Kernel-Network' and EventId in (10, 11) 3 | | extend ProcessName = getprocessname(EventData.PID) 4 | | extend DestinationIpAddress = EventData.daddr, DestinationPort = ntohs(EventData.dport) 5 | | extend Size = EventData.size 6 | | extend ProcessId = EventData.PID 7 | | summarize Packets = count(), Bytes = sum(Size) 8 | by bin(TimeCreated, 1m), ProcessId, ProcessName, DestinationIpAddress, DestinationPort 9 | 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Real-Time KQL 2 | 3 | To process data with Kusto Query Language (KQL) queries today, users generally have to upload their data to storage first and then query it. The Kql Tools eliminate this need by processing event streams with KQL queries **as events arrive, in real-time.** 4 | 5 | ![StandingQuery.jpg](StandingQuery.jpg) 6 | 7 | ## Contents 8 | 9 | * [List of tools](#Tools) 10 | * [Supported event sources](#Inputs) 11 | * [Supported event destinations](#Outputs) 12 | * [Contributing](#Contributing) 13 | 14 | 15 | 16 | ## List of Tools 17 | 18 | | **Command Line Tool** | **Python Module** | PowerShell Module | 19 | | :----------------------------------------------------------: | :------------------------------------------------: | :----------------------------------------------------------: | 20 | | [Documentation](Doc/CommandLineTool.md) | [Documentation](Doc/PythonModule.md) | [Documentation](Doc/PowerShellModule.md) | 21 | | [Downloads](https://github.com/microsoft/KqlTools/releases/) | [Downloads](https://pypi.org/project/realtimekql/) | [Downloads](https://www.powershellgallery.com/packages/RealTimeKql/) | 22 | | [Demo](https://youtu.be/utlsqlrAQgA) | [Demo](https://youtu.be/5LLpxkpm580) | [Demo](https://youtu.be/a_p_Fm-fycE) | 23 | 24 | 25 | 26 | ## Supported Event Sources 27 | 28 | In addition to processing **CSV files**, the KQL tools support the following input sources: 29 | 30 | | | Windows | Linux | 31 | | :---------------------: | :-------------------------------------------------------: | :----------------------------------------------------------: | 32 | | **OS Logs** | **WinLog** - logs seen in EventVwr or log file(s) on disk | **Syslog** - the OS log | 33 | | **High-Volume Tracing** | **Etw** - Event Tracing for Windows | **EBPF** - dynamic interception of kernel and user mode functions (*Coming soon*) | 34 | 35 | 36 | 37 | ## Supported Event Destinations 38 | 39 | | Real-Time Output | File Output | Upload Output | 40 | | :----------------------------------------------------------: | :----------------------------------------------------: | :----------------------------------------------: | 41 | | **json**- Results printed to standard output in JSON format | **json file** - Results written to file in JSON format | **adx** - Upload to Kusto (Azure Data Explorer) | 42 | | **table** - Results printed to standard output in table format | | **blob** - Upload as JSON objects to BlobStorage | 43 | 44 | 45 | 46 | ## Contributing 47 | 48 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 49 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 50 | the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. 51 | 52 | When you submit a pull request, a CLA bot will automatically determine whether you need to provide 53 | a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions 54 | provided by the bot. You will only need to do this once across all repos using our CLA. 55 | 56 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 57 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 58 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 59 | 60 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | -------------------------------------------------------------------------------- /Source/Actions/CreateReleaseAction/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Create Release' 2 | description: 'Create a release based off a tag' 3 | inputs: 4 | token: 5 | description: 'Authentication token for updating release' 6 | required: true 7 | tag_name: 8 | description: "Tag to base release on" 9 | required: true 10 | release_name: 11 | description: "Name to give new release" 12 | required: true 13 | directory: 14 | description: "Directory with assets to upload to the release after creation" 15 | required: false 16 | runs: 17 | using: 'node12' 18 | main: 'dist/index.js' -------------------------------------------------------------------------------- /Source/Actions/CreateReleaseAction/build.cmd: -------------------------------------------------------------------------------- 1 | ncc build index.js --license licenses.txt -------------------------------------------------------------------------------- /Source/Actions/CreateReleaseAction/index.js: -------------------------------------------------------------------------------- 1 | const core = require('@actions/core'); 2 | const github = require('@actions/github'); 3 | 4 | const fs = require('fs'); 5 | const util = require('util'); 6 | const readdir = util.promisify(fs.readdir); 7 | 8 | async function run() 9 | { 10 | try 11 | { 12 | // set up octokit and context information 13 | const octokit = github.getOctokit(core.getInput('token')); 14 | 15 | // getting context 16 | const owner = github.context.repo.owner; 17 | const repo = github.context.repo.repo; 18 | 19 | // create new release 20 | const tag_name = core.getInput('tag_name'); 21 | const release_name = core.getInput('release_name'); 22 | const createReleaseResponse = await octokit.repos.createRelease({ 23 | owner: owner, 24 | repo: repo, 25 | tag_name: tag_name, 26 | name: release_name 27 | }); 28 | 29 | if(createReleaseResponse.status != 201) 30 | { 31 | core.setFailed(`Problem creating release: ${createReleaseResponse.status}`); 32 | } 33 | 34 | console.log("Release created!"); 35 | 36 | // optionally upload release assets 37 | const dir = core.getInput('directory'); 38 | if(!dir) return; 39 | console.log("Uploading asset(s) to release now!"); 40 | 41 | const release = createReleaseResponse.data; 42 | const releaseAssets = await readdir(dir); 43 | 44 | for(let asset of releaseAssets) 45 | { 46 | console.log(`Uploading ${asset}...`); 47 | const uploadResponse = await octokit.repos.uploadReleaseAsset({ 48 | owner: owner, 49 | repo: repo, 50 | release_id: release.id, 51 | name: asset, 52 | data: fs.readFileSync(`${dir}\\${asset}`), 53 | origin: release.upload_url 54 | }); 55 | console.log(`Uploaded ${asset}!`); 56 | 57 | if(uploadResponse.status != 201) 58 | { 59 | core.setFailed(`Problem uploading release asset: ${asset}\nUpload response: ${uploadResponse.status}`); 60 | } 61 | } 62 | } 63 | catch(error) 64 | { 65 | core.setFailed(error.message); 66 | } 67 | } 68 | 69 | run(); -------------------------------------------------------------------------------- /Source/Actions/CreateReleaseAction/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "create-release", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/namita-prakash/create-release.git" 12 | }, 13 | "keywords": [], 14 | "author": "", 15 | "license": "ISC", 16 | "bugs": { 17 | "url": "https://github.com/namita-prakash/create-release/issues" 18 | }, 19 | "homepage": "https://github.com/namita-prakash/create-release#readme", 20 | "dependencies": { 21 | "@actions/core": "^1.2.6", 22 | "@actions/github": "^4.0.0" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Source/Actions/SetupPythonDeploymentAction/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Setup Python Deployment Action' 2 | description: 'Set up directory for python script deployment' 3 | inputs: 4 | tag: 5 | description: 'Tag that triggered this release' 6 | required: true 7 | directory: 8 | description: 'Directory to put __init__.py and setup.py in' 9 | required: true 10 | runs: 11 | using: 'node12' 12 | main: 'dist/index.js' 13 | -------------------------------------------------------------------------------- /Source/Actions/SetupPythonDeploymentAction/build.cmd: -------------------------------------------------------------------------------- 1 | ncc build index.js --license licenses.txt 2 | -------------------------------------------------------------------------------- /Source/Actions/SetupPythonDeploymentAction/index.js: -------------------------------------------------------------------------------- 1 | const core = require('@actions/core'); 2 | const github = require('@actions/github'); 3 | 4 | const fs = require('fs'); 5 | const util = require('util'); 6 | 7 | async function run() { 8 | try { 9 | // creating __init__.py files 10 | const dir = core.getInput('directory'); 11 | fs.writeFile(`${dir}\\__init__.py`, '', (err) => { 12 | if(err) { 13 | throw err; 14 | } 15 | }); 16 | 17 | // generating setup.py files 18 | const tag = core.getInput('tag'); 19 | const realtimekqlSetup = `import os 20 | import glob 21 | from setuptools import setup, find_packages 22 | 23 | setup( 24 | name = 'realtimekql', 25 | version='${tag}', 26 | install_requires=['pythonnet'], 27 | packages=['.'], 28 | data_files=[('lib', glob.glob(os.path.join('lib', '*')))], 29 | include_package_data=True, 30 | )`; 31 | 32 | fs.writeFile(`${dir}\\setup.py`, realtimekqlSetup, (err) => { 33 | if(err) { 34 | throw err; 35 | } 36 | }); 37 | 38 | } 39 | catch (err) { 40 | core.setFailed(err); 41 | } 42 | } 43 | 44 | run(); 45 | -------------------------------------------------------------------------------- /Source/Actions/SetupPythonDeploymentAction/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "setuppythondeploymentaction", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \\\"Error: no test specified\\\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@actions/core": "^1.9.1", 13 | "@actions/github": "^4.0.0" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Source/Actions/SignAction/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Signtool Action' 2 | description: 'Sign code using signtool' 3 | inputs: 4 | certificate: 5 | description: 'Base64-encoded certificate' 6 | required: true 7 | key: 8 | description: 'Key for certificate' 9 | required: true 10 | directory: 11 | description: 'Directory with binaries to sign' 12 | required: true 13 | runs: 14 | using: 'node12' 15 | main: 'dist/index.js' -------------------------------------------------------------------------------- /Source/Actions/SignAction/build.cmd: -------------------------------------------------------------------------------- 1 | ncc build index.js --license licenses.txt -------------------------------------------------------------------------------- /Source/Actions/SignAction/index.js: -------------------------------------------------------------------------------- 1 | const core = require('@actions/core'); 2 | const github = require('@actions/github'); 3 | 4 | const fs = require('fs'); 5 | const util = require('util'); 6 | 7 | const writefile = util.promisify(fs.writeFile); 8 | const exec = util.promisify(require('child_process').exec); 9 | const readdir = util.promisify(fs.readdir); 10 | 11 | async function generatepfx(cert) { 12 | try { 13 | const secretcert = Buffer.from(core.getInput('certificate'), 'base64'); 14 | await writefile(cert, secretcert); 15 | return true; 16 | } 17 | catch(err) 18 | { 19 | console.log(err); 20 | return false; 21 | } 22 | } 23 | 24 | async function getbinaries(dir) { 25 | try { 26 | const fileNames = await readdir(dir); 27 | let files = new Array(fileNames.length); 28 | await Promise.all(fileNames.map(async (fileName) => { 29 | const extension = fileName.split('.').pop(); 30 | if(extension == 'exe') { 31 | files.push(`${dir}\\${fileName}`); 32 | } 33 | })) 34 | return files; 35 | } 36 | catch (err) 37 | { 38 | console.log(err); 39 | return null; 40 | } 41 | } 42 | 43 | async function sign(files, cert) { 44 | const key = core.getInput('key'); 45 | try { 46 | await Promise.all(files.map(async (file) => { 47 | const { stdout, stderr } = await exec(`"C:\\Program Files (x86)\\Windows Kits\\10\\bin\\10.0.16299.0\\x64\\signtool.exe" sign /f "${cert}" /p ${key} /fd sha256 /tr "http://timestamp.digicert.com" /td sha256 "${file}"`); 48 | if(stderr) { 49 | console.error(`error: ${stderr}`); 50 | } 51 | else { 52 | console.log(stdout); 53 | } 54 | })); 55 | return true; 56 | } 57 | catch (err) 58 | { 59 | console.log(err); 60 | return false; 61 | } 62 | } 63 | 64 | async function run() { 65 | try { 66 | // prepare pfx for signtool 67 | const cert = `${process.env.temp}.pfx`; 68 | if (await generatepfx(cert)) { 69 | } 70 | 71 | // get binaries and sign 72 | const dir = core.getInput('directory'); 73 | const files = await getbinaries(dir); 74 | if (await sign(files, cert)) 75 | { 76 | console.log("Signed All Files!"); 77 | } 78 | } 79 | catch (err) { 80 | core.setFailed(err); 81 | } 82 | } 83 | 84 | run(); -------------------------------------------------------------------------------- /Source/Actions/SignAction/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SignAction", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "@actions/core": "^1.9.1", 14 | "@actions/github": "^4.0.0" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Source/KqlPowerShell/BaseCmdlet.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Management.Automation; 3 | using System.Threading; 4 | using RealTimeKqlLibrary; 5 | 6 | namespace KqlPowerShell 7 | { 8 | public abstract class BaseCmdlet : Cmdlet 9 | { 10 | [Parameter( 11 | Mandatory = false, 12 | ValueFromPipeline = true, 13 | ValueFromPipelineByPropertyName = true)] 14 | public string[] Queries; 15 | 16 | protected EventComponent _eventComponent; 17 | protected QueuedDictionaryOutput _output; 18 | private bool _startupError = false; 19 | 20 | // Set up event component in derived class 21 | protected abstract EventComponent SetupEventComponent(); 22 | 23 | protected override void BeginProcessing() 24 | { 25 | // Instantiating output component 26 | _output = new QueuedDictionaryOutput(); 27 | 28 | // Let child class instantiate event component 29 | _eventComponent = SetupEventComponent(); 30 | 31 | // Starting up event component 32 | if (_eventComponent == null || !_eventComponent.Start()) 33 | { 34 | WriteWarning($"ERROR! Problem starting up."); 35 | _startupError = true; 36 | } 37 | } 38 | 39 | protected override void ProcessRecord() 40 | { 41 | if (_startupError) return; 42 | 43 | // Loop runs while output is running or queue is not empty 44 | while (_output.Running || !_output.KqlOutput.IsEmpty) 45 | { 46 | int eventsPrinted = 0; 47 | if (_output.KqlOutput.TryDequeue(out IDictionary eventOutput)) 48 | { 49 | PSObject row = new PSObject(); 50 | foreach (var pair in eventOutput) 51 | { 52 | row.Properties.Add(new PSNoteProperty(pair.Key, pair.Value)); 53 | } 54 | WriteObject(row); 55 | Interlocked.Increment(ref eventsPrinted); 56 | } 57 | 58 | if (_output.Error != null) 59 | { 60 | WriteWarning(_output.Error.Message); 61 | break; 62 | } 63 | 64 | if (eventsPrinted % 10 == 0) { Thread.Sleep(20); } 65 | } 66 | } 67 | 68 | protected override void EndProcessing() 69 | { 70 | if (_output.Error != null) WriteWarning(_output.Error.Message); 71 | WriteObject("\nCompleted!\nThank you for using Real-Time KQL!"); 72 | } 73 | 74 | protected override void StopProcessing() 75 | { 76 | EndProcessing(); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /Source/KqlPowerShell/KqlCmdlets.cs: -------------------------------------------------------------------------------- 1 | using System.Management.Automation; 2 | using RealTimeKqlLibrary; 3 | 4 | namespace KqlPowerShell 5 | { 6 | [Cmdlet(VerbsCommon.Get, "CsvFileReader")] 7 | public class GetCsvFileReader : BaseCmdlet 8 | { 9 | [Parameter( 10 | Mandatory = true, 11 | Position = 0, 12 | ValueFromPipeline = true, 13 | ValueFromPipelineByPropertyName = true)] 14 | public string FilePath; 15 | 16 | protected override EventComponent SetupEventComponent() 17 | { 18 | return new CsvFileReader(FilePath, _output, Queries); 19 | } 20 | } 21 | 22 | [Cmdlet(VerbsCommon.Get, "EtwSession")] 23 | public class GetEtwSession : BaseCmdlet 24 | { 25 | [Parameter( 26 | Mandatory = true, 27 | Position = 0, 28 | ValueFromPipeline = true, 29 | ValueFromPipelineByPropertyName = true)] 30 | public string SessionName; 31 | 32 | protected override EventComponent SetupEventComponent() 33 | { 34 | return new EtwSession(SessionName, _output, Queries); 35 | } 36 | } 37 | 38 | [Cmdlet(VerbsCommon.Get, "EtlFileReader")] 39 | public class GetEtlFileReader : BaseCmdlet 40 | { 41 | [Parameter( 42 | Mandatory = true, 43 | Position = 0, 44 | ValueFromPipeline = true, 45 | ValueFromPipelineByPropertyName = true)] 46 | public string FilePath; 47 | 48 | protected override EventComponent SetupEventComponent() 49 | { 50 | return new EtlFileReader(FilePath, _output, Queries); 51 | } 52 | } 53 | 54 | [Cmdlet(VerbsCommon.Get, "WinlogRealTime")] 55 | public class GetWinlogRealTime : BaseCmdlet 56 | { 57 | [Parameter( 58 | Mandatory = true, 59 | Position = 0, 60 | ValueFromPipeline = true, 61 | ValueFromPipelineByPropertyName = true)] 62 | public string LogName; 63 | 64 | protected override EventComponent SetupEventComponent() 65 | { 66 | return new WinlogRealTime(LogName, _output, Queries); 67 | } 68 | } 69 | 70 | [Cmdlet(VerbsCommon.Get, "EvtxFileReader")] 71 | public class GetEvtxFileReader : BaseCmdlet 72 | { 73 | [Parameter( 74 | Mandatory = true, 75 | Position = 0, 76 | ValueFromPipeline = true, 77 | ValueFromPipelineByPropertyName = true)] 78 | public string FilePath; 79 | 80 | protected override EventComponent SetupEventComponent() 81 | { 82 | return new EvtxFileReader(FilePath, _output, Queries); 83 | } 84 | } 85 | 86 | [Cmdlet(VerbsCommon.Get, "SyslogFileReader")] 87 | public class GetSyslogFileReader : BaseCmdlet 88 | { 89 | [Parameter( 90 | Mandatory = true, 91 | Position = 0, 92 | ValueFromPipeline = true, 93 | ValueFromPipelineByPropertyName = true)] 94 | public string FilePath; 95 | 96 | protected override EventComponent SetupEventComponent() 97 | { 98 | return new SyslogFileReader(FilePath, _output, Queries); 99 | } 100 | } 101 | 102 | [Cmdlet(VerbsCommon.Get, "SyslogServer")] 103 | public class GetSyslogServer : BaseCmdlet 104 | { 105 | [Parameter( 106 | Mandatory = false, 107 | ValueFromPipeline = true, 108 | ValueFromPipelineByPropertyName = true)] 109 | public string NetworkAdapterName; 110 | 111 | [Parameter( 112 | Mandatory = false, 113 | ValueFromPipeline = true, 114 | ValueFromPipelineByPropertyName = true)] 115 | public int UdpPort = 514; 116 | 117 | protected override EventComponent SetupEventComponent() 118 | { 119 | return new SyslogServer(NetworkAdapterName, UdpPort, _output, Queries); 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /Source/KqlPowerShell/KqlPowerShell.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | net472 5 | KqlPowerShell 6 | 7 | 8 | 9 | 10 | All 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | Always 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /Source/KqlPowerShell/QueuedDictionaryOutput.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Concurrent; 3 | using System.Collections.Generic; 4 | using System.Reactive.Kql.CustomTypes; 5 | using RealTimeKqlLibrary; 6 | 7 | namespace KqlPowerShell 8 | { 9 | public class QueuedDictionaryOutput : IOutput 10 | { 11 | public ConcurrentQueue> KqlOutput; 12 | public Exception Error { get; private set; } 13 | public bool Running { get; private set; } 14 | private const int MAX_EVENTS = 1000000; 15 | 16 | public QueuedDictionaryOutput() 17 | { 18 | KqlOutput = new ConcurrentQueue>(); 19 | Running = true; 20 | } 21 | 22 | public void KqlOutputAction(KqlOutput obj) 23 | { 24 | OutputAction(obj.Output); 25 | } 26 | 27 | public void OutputAction(IDictionary obj) 28 | { 29 | if(Running) 30 | { 31 | if(KqlOutput.Count > MAX_EVENTS) 32 | { 33 | // Clear queue if user hasn't tried to dequeue last MAX_EVENTS events 34 | KqlOutput = new ConcurrentQueue>(); 35 | } 36 | try 37 | { 38 | KqlOutput.Enqueue(obj); 39 | } 40 | catch (Exception ex) 41 | { 42 | OutputError(ex); 43 | } 44 | } 45 | } 46 | 47 | public void OutputError(Exception ex) 48 | { 49 | // Stop enqueueing new events 50 | Running = false; 51 | Error = ex; 52 | } 53 | 54 | public void OutputCompleted() 55 | { 56 | // Stop enqueueing new events 57 | Running = false; 58 | } 59 | 60 | public void Stop() 61 | { 62 | // can add code to exit process here if needed 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /Source/KqlPowerShell/README.md: -------------------------------------------------------------------------------- 1 | # Real-Time KQL for PowerShell 2 | 3 | To process data with Kusto Query Language (KQL) queries today, users generally have to upload their data to storage first and then query it. The Kql Tools eliminate this need by processing event streams with KQL queries **as events arrive, in real-time.** 4 | 5 | 6 | 7 | ## Tracing ETW TCP Events 8 | 9 | This walkthrough demonstrates how to use the Real-Time KQL PowerShell Module to explore TCP events emitted by an Event Tracing for Windows (ETW) provider. This walkthrough will use an **Administrator PowerShell** window throughout. 10 | 11 | 12 | 13 | ### Start an ETW Trace: 14 | 15 | logman is a utility that allows you to start an Event Trace Session for a specific ETW provider or set of providers. Run this command to start an event trace session for the Etw TCP provider: 16 | 17 | ``` 18 | logman.exe create trace tcp -rt -nb 2 2 -bs 1024 -p 'Microsoft-Windows-Kernel-Network' 0xffffffffffffffff -ets 19 | ``` 20 | 21 | By running `create trace tcp`, this session has been named "tcp". 22 | 23 | 24 | 25 | ### Import RealTimeKql: 26 | 27 | ``` 28 | Import-Module RealTimeKql 29 | ``` 30 | 31 | 32 | 33 | ### Look at raw TCP data: 34 | 35 | Run the Get-EtwSession cmdlet to see Etw data. Here "tcp" is the session name used earlier in the logman command to create the event trace session: 36 | 37 | ``` 38 | Get-EtwSession tcp 39 | ``` 40 | 41 | 42 | 43 | ### Simplify TCP event output: 44 | 45 | Store this query in a .kql file (for this exercise, we'll assume the file is named **query.kql**): 46 | 47 | ``` 48 | EtwTcp 49 | | where Provider == "Microsoft-Windows-Kernel-Network" 50 | | where EventId in (10, 11) 51 | | extend ProcessName = getprocessname(EventData.PID) 52 | | extend Source = strcat(EventData.saddr, ":", ntohs(EventData.sport)) 53 | | extend Destination = strcat(EventData.daddr, ":", ntohs(EventData.dport)) 54 | | project Provider, Source, Destination, Opcode, ProcessName 55 | ``` 56 | 57 | You can now use this query to simplify the output you see: 58 | 59 | ``` 60 | Get-EtwSession tcp -Query | Format-Table 61 | ``` 62 | 63 | 64 | 65 | ### Aggregate TCP events into 30-second intervals: 66 | 67 | To reduce the volume of events printing to your screen, you can apply a different query that aggregates events into 30-second windows. Store this query in a .kql file (for this exercise, we'll assume the file is named **summarize.kql**): 68 | 69 | ``` 70 | EtwTcp 71 | | where EventId in (10, 11) 72 | | extend ProcessName = getprocessname(EventData.PID) 73 | | extend Source = EventData.saddr, SourcePort = ntohs(EventData.sport) 74 | | extend Destination = EventData.daddr, DestinationPort = ntohs(EventData.dport) 75 | | extend Size = EventData.size 76 | | extend ProcessId = EventData.PID 77 | | summarize Count = count(), Bytes = sum(Size) by bin(TimeCreated, 30s), SourceIpAddress, SourcePort, DestinationIpAddress, DestinationPort, EventId, ProcessId, ProcessName 78 | ``` 79 | 80 | Pass in this new query to the Get-EtwSession cmdlet: 81 | 82 | ``` 83 | Get-EtwSession tcp -Query | Format-Table 84 | ``` 85 | 86 | 87 | 88 | ### Stop an ETW Trace: 89 | 90 | Once you are done exploring events, you can stop the Event Trace Session you started with logman by running the following command: 91 | 92 | ``` 93 | logman.exe stop tcp -ets 94 | ``` -------------------------------------------------------------------------------- /Source/KqlPowerShell/RealTimeKql.psd1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/KqlTools/890ab51c8c4032a94762a842c6f6e3214e4a5912/Source/KqlPowerShell/RealTimeKql.psd1 -------------------------------------------------------------------------------- /Source/KqlPython/README.md: -------------------------------------------------------------------------------- 1 | # Real-Time KQL for Python 2 | 3 | To process data with Kusto Query Language (KQL) queries today, users generally have to upload their data to storage first and then query it. The Kql Tools eliminate this need by processing event streams with KQL queries **as events arrive, in real-time.** 4 | 5 | 6 | 7 | ## Usage 8 | 9 | Real-Time KQL is broken up into three parts: the output, the query, and the input. 10 | 11 | ### The Output 12 | 13 | Real-Time KQL for Python has a `PythonOutput` class that allows you to customize what happens to events when they are outputted. The simplest usage of the `PythonOutput` class is to instantiate it with no parameters. This will print events to console in JSON format: 14 | 15 | ``` 16 | >>> from realtimekql import * 17 | >>> o = PythonOutput() 18 | ``` 19 | 20 | To customize the output, you can pass in any Python function that takes a dictionary as the only parameter to the `PythonOutput` class. For example, this function stores events in a list to use them later: 21 | 22 | ``` 23 | >>> events = [] 24 | >>> def storeEvents(event): 25 | ... events.append(event) 26 | ... 27 | >>> from realtimekql import * 28 | >>> o = PythonOutput(storeEvents) 29 | ``` 30 | 31 | The `PythonAdxOutput` class allows you to ingest data to an Azure Data Explorer (Kusto) table through queued ingestion. The class can be instantiated as follows: 32 | 33 | ``` 34 | >>> from realtimekql import * 35 | >>> o = PythonAdxOutput("YourCluster.kusto.windows.net", "YourDatabase", "YourTable", "YourClientId", "YourClientSecret", "YourAuthorityId", resetTable=True) 36 | ``` 37 | 38 | 39 | 40 | ### The Query 41 | 42 | You can optionally pass a .kql query into Real-Time KQL to filter, transform, and enrich your events before they even reach the output stage. 43 | 44 | 45 | 46 | ### The Input 47 | 48 | Real-Time KQL supports various real-time and file input sources. Each input class takes a unique set of arguments, an instance of one of the output classes, as well as an optional path to a query file. This prints real-time Etw TCP events to console in JSON format: 49 | 50 | ``` 51 | >>> from realtimekql import * 52 | >>> o = PythonOutput() 53 | >>> e = EtwSession("tcp", o) 54 | >>> e.Start() 55 | ``` 56 | 57 | Here are all the supported input options and how to use them: 58 | 59 | ``` 60 | EtwSession(sessionName, o, q) 61 | EtlFileReader(filePath, o, q) 62 | WinlogRealTime(logName, o, q) 63 | EvtxFileReader(filePath, o, q) 64 | CsvFileReader(filePath, o, q) 65 | ``` 66 | 67 | The variables `o` and `q` represent the output part and the query part respectively. The query part is optional and can be left out. 68 | 69 | -------------------------------------------------------------------------------- /Source/KqlPython/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/KqlTools/890ab51c8c4032a94762a842c6f6e3214e4a5912/Source/KqlPython/__init__.py -------------------------------------------------------------------------------- /Source/KqlPython/setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | from setuptools import setup, find_packages 4 | 5 | import kqlpythondir 6 | 7 | with open(os.path.join(kqlpythondir.directory, 'VERSION.txt'), encoding='utf-8') as f: 8 | version = f.read() 9 | 10 | with open(os.path.join(kqlpythondir.directory, 'README.md'), encoding='utf-8') as f: 11 | long_description = f.read() 12 | 13 | setup( 14 | name = 'realtimekql', 15 | version=version, 16 | author='CDOC Engineering Open Source', 17 | author_email='CDOCEngOpenSourceAdm@microsoft.com', 18 | url='https://github.com/microsoft/kqltools', 19 | description='A module for exploring real-time streams of events', 20 | long_description=long_description, 21 | long_description_content_type='text/markdown', 22 | install_requires=['pythonnet', 'pandas', 'azure.kusto.data', 'azure.kusto.ingest'], 23 | packages=['.'], 24 | data_files=[('lib', glob.glob(os.path.join('lib', '*')))], 25 | include_package_data=True, 26 | classifiers=[ 27 | 'Environment :: Console', 28 | 'Intended Audience :: End Users/Desktop', 29 | 'Intended Audience :: Developers', 30 | 'License :: OSI Approved :: Apache Software License', 31 | 'Natural Language :: English', 32 | 'Operating System :: Microsoft :: Windows', 33 | 'Programming Language :: C#', 34 | 'Programming Language :: Python :: 3', 35 | 'Topic :: Security', 36 | 'Topic :: System :: Logging', 37 | 'Topic :: System :: Monitoring', 38 | 'Topic :: System :: Networking', 39 | 'Topic :: System :: Networking :: Monitoring', 40 | 'Topic :: System :: Operating System', 41 | 'Topic :: System :: Operating System Kernels', 42 | ], 43 | ) 44 | -------------------------------------------------------------------------------- /Source/KqlTools.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | # Visual Studio Version 16 4 | VisualStudioVersion = 16.0.30204.135 5 | MinimumVisualStudioVersion = 10.0.40219.1 6 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Syslog", "Microsoft.Syslog\Microsoft.Syslog.csproj", "{E6AC40D2-EB4D-4AA5-B071-E01674E3A491}" 7 | EndProject 8 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RealTimeKqlLibrary", "RealTimeKqlLibrary\RealTimeKqlLibrary.csproj", "{33323763-616B-4352-8B98-3A5B0F72B0B6}" 9 | EndProject 10 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RealTimeKql", "RealTimeKql\RealTimeKql.csproj", "{088394A8-4018-4E22-B9B7-C370B9E64F8B}" 11 | EndProject 12 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RealTimeKqlTests", "RealTimeKqlTests\RealTimeKqlTests.csproj", "{DB63AD1A-5BB1-4C41-AA43-60FB34BE5448}" 13 | EndProject 14 | Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KqlPowerShell", "KqlPowerShell\KqlPowerShell.csproj", "{9B17C34B-7C9E-45C6-AB5C-9CC1CB0E1A73}" 15 | EndProject 16 | Global 17 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 18 | Debug|Any CPU = Debug|Any CPU 19 | Debug|x64 = Debug|x64 20 | Release|Any CPU = Release|Any CPU 21 | Release|x64 = Release|x64 22 | EndGlobalSection 23 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 24 | {E6AC40D2-EB4D-4AA5-B071-E01674E3A491}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 25 | {E6AC40D2-EB4D-4AA5-B071-E01674E3A491}.Debug|Any CPU.Build.0 = Debug|Any CPU 26 | {E6AC40D2-EB4D-4AA5-B071-E01674E3A491}.Debug|x64.ActiveCfg = Debug|Any CPU 27 | {E6AC40D2-EB4D-4AA5-B071-E01674E3A491}.Debug|x64.Build.0 = Debug|Any CPU 28 | {E6AC40D2-EB4D-4AA5-B071-E01674E3A491}.Release|Any CPU.ActiveCfg = Release|Any CPU 29 | {E6AC40D2-EB4D-4AA5-B071-E01674E3A491}.Release|Any CPU.Build.0 = Release|Any CPU 30 | {E6AC40D2-EB4D-4AA5-B071-E01674E3A491}.Release|x64.ActiveCfg = Release|Any CPU 31 | {E6AC40D2-EB4D-4AA5-B071-E01674E3A491}.Release|x64.Build.0 = Release|Any CPU 32 | {33323763-616B-4352-8B98-3A5B0F72B0B6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 33 | {33323763-616B-4352-8B98-3A5B0F72B0B6}.Debug|Any CPU.Build.0 = Debug|Any CPU 34 | {33323763-616B-4352-8B98-3A5B0F72B0B6}.Debug|x64.ActiveCfg = Debug|Any CPU 35 | {33323763-616B-4352-8B98-3A5B0F72B0B6}.Debug|x64.Build.0 = Debug|Any CPU 36 | {33323763-616B-4352-8B98-3A5B0F72B0B6}.Release|Any CPU.ActiveCfg = Release|Any CPU 37 | {33323763-616B-4352-8B98-3A5B0F72B0B6}.Release|Any CPU.Build.0 = Release|Any CPU 38 | {33323763-616B-4352-8B98-3A5B0F72B0B6}.Release|x64.ActiveCfg = Release|Any CPU 39 | {33323763-616B-4352-8B98-3A5B0F72B0B6}.Release|x64.Build.0 = Release|Any CPU 40 | {088394A8-4018-4E22-B9B7-C370B9E64F8B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 41 | {088394A8-4018-4E22-B9B7-C370B9E64F8B}.Debug|Any CPU.Build.0 = Debug|Any CPU 42 | {088394A8-4018-4E22-B9B7-C370B9E64F8B}.Debug|x64.ActiveCfg = Debug|Any CPU 43 | {088394A8-4018-4E22-B9B7-C370B9E64F8B}.Debug|x64.Build.0 = Debug|Any CPU 44 | {088394A8-4018-4E22-B9B7-C370B9E64F8B}.Release|Any CPU.ActiveCfg = Release|Any CPU 45 | {088394A8-4018-4E22-B9B7-C370B9E64F8B}.Release|Any CPU.Build.0 = Release|Any CPU 46 | {088394A8-4018-4E22-B9B7-C370B9E64F8B}.Release|x64.ActiveCfg = Release|Any CPU 47 | {088394A8-4018-4E22-B9B7-C370B9E64F8B}.Release|x64.Build.0 = Release|Any CPU 48 | {DB63AD1A-5BB1-4C41-AA43-60FB34BE5448}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 49 | {DB63AD1A-5BB1-4C41-AA43-60FB34BE5448}.Debug|Any CPU.Build.0 = Debug|Any CPU 50 | {DB63AD1A-5BB1-4C41-AA43-60FB34BE5448}.Debug|x64.ActiveCfg = Debug|Any CPU 51 | {DB63AD1A-5BB1-4C41-AA43-60FB34BE5448}.Debug|x64.Build.0 = Debug|Any CPU 52 | {DB63AD1A-5BB1-4C41-AA43-60FB34BE5448}.Release|Any CPU.ActiveCfg = Release|Any CPU 53 | {DB63AD1A-5BB1-4C41-AA43-60FB34BE5448}.Release|Any CPU.Build.0 = Release|Any CPU 54 | {DB63AD1A-5BB1-4C41-AA43-60FB34BE5448}.Release|x64.ActiveCfg = Release|Any CPU 55 | {DB63AD1A-5BB1-4C41-AA43-60FB34BE5448}.Release|x64.Build.0 = Release|Any CPU 56 | {9B17C34B-7C9E-45C6-AB5C-9CC1CB0E1A73}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 57 | {9B17C34B-7C9E-45C6-AB5C-9CC1CB0E1A73}.Debug|Any CPU.Build.0 = Debug|Any CPU 58 | {9B17C34B-7C9E-45C6-AB5C-9CC1CB0E1A73}.Debug|x64.ActiveCfg = Debug|Any CPU 59 | {9B17C34B-7C9E-45C6-AB5C-9CC1CB0E1A73}.Debug|x64.Build.0 = Debug|Any CPU 60 | {9B17C34B-7C9E-45C6-AB5C-9CC1CB0E1A73}.Release|Any CPU.ActiveCfg = Release|Any CPU 61 | {9B17C34B-7C9E-45C6-AB5C-9CC1CB0E1A73}.Release|Any CPU.Build.0 = Release|Any CPU 62 | {9B17C34B-7C9E-45C6-AB5C-9CC1CB0E1A73}.Release|x64.ActiveCfg = Release|Any CPU 63 | {9B17C34B-7C9E-45C6-AB5C-9CC1CB0E1A73}.Release|x64.Build.0 = Release|Any CPU 64 | EndGlobalSection 65 | GlobalSection(SolutionProperties) = preSolution 66 | HideSolutionNode = FALSE 67 | EndGlobalSection 68 | GlobalSection(ExtensibilityGlobals) = postSolution 69 | SolutionGuid = {3805DB70-EC3F-4D3C-A7CE-FC89532607FC} 70 | EndGlobalSection 71 | EndGlobal 72 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Internals/BatchingQueue.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Internals 8 | { 9 | using System; 10 | using System.Collections.Generic; 11 | using System.Diagnostics; 12 | using System.Runtime.CompilerServices; 13 | using System.Threading; 14 | 15 | /// 16 | /// Implements a batching queue - concurrent no-lock enqueue-one; dequeue many with lock. 17 | /// 18 | [DebuggerDisplay("Count = {Count}")] 19 | public sealed class BatchingQueue : IObserver 20 | { 21 | 22 | public int Count => _count; 23 | 24 | class Node 25 | { 26 | public T Item; 27 | public Node Next; 28 | } 29 | 30 | // We use Interlocked operations when accessing last-in (to push items); 31 | // we use lock when accessing _lastOut element (dequeuing multiple items) 32 | volatile Node _lastIn; //must be marked as volatile for interlocked ops 33 | Node _lastOut; 34 | object _dequeueLock = new object(); 35 | volatile int _count; 36 | 37 | public BatchingQueue() 38 | { 39 | // There's always at least one empty node in linked list; so empty queue holds just one node. 40 | // this is necessary to avoid problem with pushing first node (or popping the last one) - when you have 41 | // to modify both pointers to start and end of the list from null to this first element. 42 | _lastIn = _lastOut = new Node(); 43 | } 44 | 45 | public void Enqueue(T item) 46 | { 47 | // 1. Get node from pool or create new one 48 | var newNode = NodePoolTryPop() ?? new Node(); 49 | newNode.Item = item; 50 | // 2. Quick path 51 | var oldLastIn = _lastIn; 52 | if (Interlocked.CompareExchange(ref _lastIn, newNode, oldLastIn) == oldLastIn) 53 | oldLastIn.Next = newNode; 54 | else 55 | EnqueueSlowPath(newNode); 56 | Interlocked.Increment(ref _count); 57 | } 58 | 59 | // Same as Enqueue but in a loop with spin 60 | private void EnqueueSlowPath(Node newNode) 61 | { 62 | var spinWait = new SpinWait(); 63 | Node oldLastIn; 64 | do 65 | { 66 | spinWait.SpinOnce(); 67 | oldLastIn = _lastIn; 68 | } while (Interlocked.CompareExchange(ref _lastIn, newNode, oldLastIn) != oldLastIn); 69 | oldLastIn.Next = newNode; 70 | } 71 | 72 | 73 | public IList DequeueMany(int maxCount = int.MaxValue) 74 | { 75 | // iterate over list starting with _lastOut 76 | var list = new List(maxCount); 77 | lock (_dequeueLock) 78 | { 79 | while (_lastOut.Next != null && list.Count < maxCount) 80 | { 81 | // save the ref to the node to return it to the pool at the end 82 | var savedLastOut = _lastOut; 83 | // Advance _lastOut, copy item to result list. 84 | _lastOut = _lastOut.Next; 85 | list.Add(_lastOut.Item); 86 | _lastOut.Item = default(T); //clear the ref to data 87 | Interlocked.Decrement(ref _count); 88 | NodePoolTryPush(savedLastOut); // return the node to the pool 89 | } 90 | return list; 91 | } //lock 92 | } //method 93 | 94 | #region Node pooling 95 | // We pool/reuse Node objects; we save nodes in a simple concurrent stack. 96 | // The stack is not 100% reliable - it might fail occasionally when pushing/popping up nodes 97 | volatile Node _nodePoolHead; 98 | 99 | private Node NodePoolTryPop() 100 | { 101 | var head = _nodePoolHead; 102 | if (head == null) // stack is empty 103 | return null; 104 | if (Interlocked.CompareExchange(ref _nodePoolHead, head.Next, head) == head) 105 | { 106 | head.Next = null; 107 | return head; 108 | } 109 | // Node pool is not reliable (push and pop), 110 | // Hypotethically this may result in pool growth over time: let's say all pushes succeed, but some rare pops fail. 111 | // To prevent this from ever happening, we drop the entire pool if we ever fail to pop. 112 | // This is an EXTREMELY rare event, and if it happens - no impact, just extra objects for GC to collect 113 | _nodePoolHead = null; //drop the pool 114 | return null; 115 | } 116 | 117 | private void NodePoolTryPush(Node node) 118 | { 119 | node.Item = default(T); 120 | node.Next = _nodePoolHead; 121 | // we make just one attempt; if it fails, we don't care - node will be GC-d 122 | Interlocked.CompareExchange(ref _nodePoolHead, node, node.Next); 123 | } 124 | #endregion 125 | 126 | #region IObserver implementation 127 | void IObserver.OnCompleted() 128 | { 129 | } 130 | 131 | void IObserver.OnError(Exception error) 132 | { 133 | } 134 | 135 | void IObserver.OnNext(T value) 136 | { 137 | this.Enqueue(value); 138 | } 139 | #endregion 140 | 141 | } //class 142 | } 143 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Internals/Observable.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Internals 8 | { 9 | using System; 10 | using System.Collections.Generic; 11 | using System.Linq; 12 | using System.Threading; 13 | 14 | /// 15 | /// Base class for components implementing the interface. 16 | /// 17 | /// The data type. 18 | /// Provides sync and async broadcast mode. 19 | public class Observable : IObservable 20 | { 21 | /// Constructs a new instance. 22 | public Observable() 23 | { 24 | } 25 | 26 | // We use copy, add/remove, replace method when adding/removing subscriptions 27 | // As a benefit, when we call OnNext on each subscriber, we iterate the list without locks 28 | private IList _subscriptions = new List(); 29 | private readonly object _lock = new object(); 30 | 31 | #region IObservable implementation 32 | 33 | /// Subscribes an observer. 34 | /// An observer instance. 35 | /// A disposable subscription object. Disposing the object cancels the subscription. 36 | public IDisposable Subscribe(IObserver observer) 37 | { 38 | return AddSubscription(observer); 39 | } 40 | 41 | #endregion 42 | 43 | /// Unsubscribes an observer. 44 | /// An observer instance. 45 | public void Unsubscribe(IObserver observer) 46 | { 47 | RemoveSubscription(observer); 48 | } 49 | 50 | /// Broadcasts an data. Calls the OnNext method of all subscribed observers. 51 | /// An data to broadcast. 52 | /// The subscribers are notified according to the broadcast mode - in sync or async manner. 53 | public void Broadcast(T item) 54 | { 55 | var sList = _subscriptions; 56 | for (int i = 0; i < sList.Count; i++) 57 | { 58 | sList[i].Observer.OnNext(item); 59 | } 60 | } 61 | 62 | /// Broadcasts an error. Calls OnError method of all subscribers passing the exception as a parameter. 63 | /// The exception to broadcast. 64 | protected void BroadcastError(Exception error) 65 | { 66 | var sList = _subscriptions; 67 | for (int i = 0; i < sList.Count; i++) 68 | { 69 | sList[i].Observer.OnError(error); 70 | } 71 | } 72 | 73 | /// Calls OnCompleted method of all subscribers. 74 | protected void BroadcastOnCompleted() 75 | { 76 | var sList = _subscriptions; 77 | for (int i = 0; i < sList.Count; i++) 78 | { 79 | sList[i].Observer.OnCompleted(); 80 | } 81 | } 82 | 83 | // Subscription list operations 84 | // We use copy, add/remove, replace method when adding/removing subscriptions 85 | // As a benefit, when we call OnNext on each subscriber, we iterate the list without locks 86 | private Subscription AddSubscription(IObserver observer) 87 | { 88 | lock (_lock) 89 | { 90 | var newList = new List(_subscriptions); 91 | var subscr = new Subscription() { Observable = this, Observer = observer }; 92 | newList.Add(subscr); 93 | Interlocked.Exchange(ref _subscriptions, newList); 94 | return subscr; 95 | } 96 | } 97 | 98 | private void RemoveSubscription(IObserver observer) 99 | { 100 | lock (_lock) 101 | { 102 | var newList = new List(_subscriptions); 103 | var subscr = newList.FirstOrDefault(s => s.Observer == observer); 104 | if (subscr != null) 105 | { 106 | newList.Remove(subscr); 107 | Interlocked.Exchange(ref _subscriptions, newList); 108 | } 109 | } 110 | } 111 | 112 | /// Represents a subscription for an observable source. 113 | internal class Subscription : IDisposable 114 | { 115 | public Observable Observable; 116 | public IObserver Observer; 117 | 118 | public void Dispose() 119 | { 120 | Observable.Unsubscribe(Observer); 121 | } 122 | } 123 | 124 | } //class 125 | } -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Microsoft.Syslog.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | Library 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Model/Enums.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Model 8 | { 9 | 10 | public enum PayloadType 11 | { 12 | 13 | /// String payload is missing syslog prefix <n>. 14 | NotSyslog, 15 | 16 | /// RFC-5424 compliant entry. 17 | Rfc5424, 18 | 19 | /// Old format, aka BSD format, described in RFC-3164; or close to it. 20 | Rfc3164, 21 | /// List of key-value pairs, Sophos Web app firewall 22 | /// (https://docs.sophos.com/nsg/sophos-firewall/v16058/Help/en-us/webhelp/onlinehelp/index.html#page/onlinehelp/WAFLogs.html). 23 | KeyValuePairs, 24 | 25 | /// Plain text not following particular standard. 26 | PlainText, 27 | } 28 | 29 | 30 | 31 | // Defined in RFC-5424 32 | 33 | public enum Severity : byte 34 | { 35 | Emergency = 0, 36 | Alert, 37 | Critical, 38 | Error, 39 | Warning, 40 | Notice, 41 | Informational, 42 | Debug 43 | } 44 | 45 | public enum Facility : byte 46 | { 47 | Kernel = 0, 48 | UserLevel, 49 | MailSystem, 50 | SystemDaemons, 51 | Authorization, 52 | Syslog, 53 | Printer, 54 | News, 55 | Uucp, 56 | Clock, 57 | SecurityAuth, 58 | Ftp, 59 | Ntp, 60 | LogAudit, 61 | LogAlert, 62 | ClockDaemon, 63 | Local0, 64 | Local1, 65 | Local2, 66 | Local3, 67 | Local4, 68 | Local5, 69 | Local6, 70 | Local7 71 | } 72 | 73 | 74 | } 75 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Model/SyslogEntry.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Model 8 | { 9 | using System; 10 | using System.Collections.Generic; 11 | using System.Diagnostics; 12 | using System.Net; 13 | 14 | [DebuggerDisplay("{Header}")] 15 | public class SyslogEntry 16 | { 17 | public PayloadType PayloadType; 18 | public Facility Facility; 19 | public Severity Severity; 20 | public SyslogHeader Header; 21 | public string Message; 22 | 23 | /// The StructuredData element for entries following RFC-5424 spec. It is a list of named elements, 24 | /// each having a list of name-value pairs. 25 | /// Keys inside element can be reapeated (see RFC 5424, example with ip parameter), so element value is a list of pairs, not dictionary. 26 | public IDictionary> StructuredData = new Dictionary>(); 27 | 28 | // same here, names can be repeated; 29 | /// Data extracted from text message sections using various extraction methods, mostly pattern-matching. Keys/names can be repeated, 30 | /// so the data is represented as list of key-value pairs, not dictionary. 31 | public IList ExtractedData = new List(); 32 | 33 | /// All data, parsed and extracted, represented as a Dictionary. Values from the and 34 | /// are combined in this single dictionary. 35 | /// Each value in a dictionary is either a single value, or an array of strings, if there is more than one value. 36 | /// The exception is IPv4 and IPv6 entries, which are always arrays, even if there is just one value. This is done for conveniences 37 | /// of querying the data in databases like Kusto. 38 | public IDictionary AllData = new Dictionary(); 39 | 40 | public SyslogEntry() { 41 | Header = new SyslogHeader(); 42 | } 43 | 44 | public SyslogEntry(Facility facility, Severity severity, DateTime? timestamp = null, string hostName = null, 45 | string appName = null, string procId = null, string msgId = null, string message = null) 46 | { 47 | PayloadType = PayloadType.Rfc5424; 48 | Facility = facility; 49 | Severity = severity; 50 | timestamp = timestamp ?? DateTime.UtcNow; 51 | Header = new SyslogHeader() 52 | { 53 | Timestamp = timestamp, HostName = hostName, 54 | AppName = appName, ProcId = procId, MsgId = msgId //Version should always be 1 55 | }; 56 | Message = message; 57 | } 58 | public override string ToString() => Header?.ToString(); 59 | } 60 | 61 | public class SyslogHeader 62 | { 63 | public DateTime? Timestamp; 64 | public string HostName; 65 | public string AppName; 66 | public string ProcId; 67 | public string MsgId; 68 | public override string ToString() => $"{Timestamp} host:{HostName} app: {AppName}"; 69 | } 70 | 71 | [DebuggerDisplay("{Name}={Value}")] 72 | public class NameValuePair 73 | { 74 | public string Name; 75 | public string Value; 76 | } 77 | 78 | public class UdpPacket 79 | { 80 | public DateTime ReceivedOn; 81 | public IPAddress SourceIpAddress; 82 | public byte[] Data; 83 | } 84 | 85 | [DebuggerDisplay("{Payload}")] 86 | public class ServerSyslogEntry 87 | { 88 | public UdpPacket UdpPacket; 89 | public string Payload; //entire syslog string 90 | public SyslogEntry Entry; 91 | public IList ParseErrorMessages; 92 | public bool Ignore; // filtered out 93 | 94 | public override string ToString() => $"{Payload}"; 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Model/SyslogExtensions.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Model 8 | { 9 | using System; 10 | using System.Collections.Generic; 11 | using System.Linq; 12 | using System.Text; 13 | 14 | public static class SyslogExtensions 15 | { 16 | public static void AddRange(this IList bucket, IList prms) 17 | { 18 | foreach (var prm in prms) 19 | bucket.Add(prm); 20 | } 21 | 22 | public static void Add(this IList bucket, string name, string value) 23 | { 24 | bucket.Add(new NameValuePair() { Name = name, Value = value }); 25 | } 26 | 27 | public static void BuildAllDataDictionary(this SyslogEntry entry) 28 | { 29 | // entry.StructuredData is RFC-5424 structured data; 30 | // ExtractedData is key-values extracted from syslog body for other formats 31 | var allParams = new List(); 32 | allParams.AddRange(entry.ExtractedData); 33 | 34 | // This is RFC-5424 specific data, structured in its own way - we take all parameters (kv pairs) from there 35 | var structData = entry.StructuredData; 36 | if (structData != null) 37 | { 38 | var structDataParams = structData.SelectMany(de => de.Value).ToList(); 39 | allParams.AddRange(structDataParams); 40 | } 41 | 42 | // now convert to dictionary; first group param values by param name 43 | var grouped = allParams.GroupBy(p => p.Name, p => p.Value).ToDictionary(g => g.Key, g => g.ToArray()); 44 | // copy to final dictionary - keep multiple values as an array; 45 | // for single value put it as a single object (not object[1]); - except for IP addresses - these are always arrays, even if there's only one 46 | entry.AllData.Clear(); 47 | foreach (var kv in grouped) 48 | { 49 | if (kv.Value.Length > 1 || kv.Key == "IPv4" || kv.Key == "IPv6") 50 | { 51 | entry.AllData[kv.Key] = kv.Value; //array 52 | } 53 | else 54 | entry.AllData[kv.Key] = kv.Value[0]; //single value 55 | } 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/Extractors/IValuesExtractor.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using Microsoft.Syslog.Model; 10 | using System.Collections.Generic; 11 | 12 | /// 13 | /// A value extractor is a component for extracting values from free-form message part of syslog, for ex: IP addresses. 14 | /// The extracted values are placed into the StructuredData dictionary of the syslog entry. 15 | /// 16 | public interface IValuesExtractor 17 | { 18 | IList ExtractValues(ParserContext context); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/Extractors/KeywordValuesExtractorBase.cs: -------------------------------------------------------------------------------- 1 | // // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using System; 10 | using System.Collections.Generic; 11 | using System.Text; 12 | using Microsoft.Syslog.Model; 13 | 14 | /// Extracts selected keywords and following values from a text message. 15 | public abstract class KeywordValuesExtractorBase : IValuesExtractor 16 | { 17 | public abstract IList ExtractValues(ParserContext ctx); 18 | 19 | protected bool TryExtractAndAdd(string message, string keyword, IList prms, bool grabAll = false) 20 | { 21 | try 22 | { 23 | if (TryExtractValue(message, keyword, out var value, grabAll)) 24 | { 25 | prms.Add(new NameValuePair() { Name = keyword, Value = value }); 26 | return true; 27 | } 28 | } 29 | catch (Exception ex) 30 | { 31 | var err = $"Error while extracting keyword {keyword} from message: {ex.Message}"; 32 | prms.Add( new NameValuePair() {Name = "ExtractorError", Value = err }); 33 | } 34 | return false; 35 | } 36 | 37 | protected bool TryExtractValue(string message, string keyword, out string value, bool grabAll = false) 38 | { 39 | value = null; 40 | var pos = message.IndexOf(keyword, StringComparison.OrdinalIgnoreCase); 41 | if (pos < 0) 42 | return false; 43 | // if preceeded by letter or digit - this is partial match, ignore it 44 | if (pos > 0 && char.IsLetterOrDigit(message[pos-1])) 45 | return false; 46 | pos += keyword.Length; 47 | if (pos >= message.Length - 2) 48 | return false; 49 | // if followed by letter or digit - this is partial match, ignore it 50 | if (char.IsLetterOrDigit(message[pos])) 51 | return false; 52 | // skip special symbols like = 53 | var valueStart = pos = message.Skip(pos, ' ', ':', '='); 54 | int valueEnd = -1; 55 | var startCh = message[valueStart]; 56 | if (startCh == '"' || startCh == '\'') 57 | { 58 | valueStart++; 59 | valueEnd = message.SkipUntil(valueStart, startCh); 60 | } 61 | else 62 | { 63 | valueEnd = grabAll? message.Length : message.SkipUntil(valueStart, ' ', ',', ';', ']', ')'); 64 | } 65 | 66 | if (valueEnd <= valueStart) 67 | { 68 | return false; 69 | } 70 | value = message.Substring(valueStart, valueEnd - valueStart); 71 | return true; 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/ParserContext.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using Microsoft.Syslog.Model; 10 | using System.Collections.Generic; 11 | using System.Diagnostics; 12 | 13 | [DebuggerDisplay("P:{Position}, ch: {Current}")] 14 | public class ParserContext 15 | { 16 | public string Text; 17 | public int Position; 18 | public string Prefix; // standard prefix identifying 'syslog' message 19 | public SyslogEntry Entry; 20 | public readonly List ErrorMessages = new List(); 21 | 22 | public char Current => Position < Text.Length ? Text[Position] : '\0'; 23 | public char CharAt(int position) => this.Text[position]; 24 | public bool Eof() => this.Position >= this.Text.Length; 25 | 26 | public ParserContext(string text) 27 | { 28 | Text = text.CutOffBOM(); 29 | 30 | } 31 | 32 | public void AddError(string message) 33 | { 34 | ErrorMessages.Add($"{message} (near {this.Position})"); 35 | } 36 | 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/ParserContextExtensions.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using System; 10 | using System.Collections.Generic; 11 | using System.Linq; 12 | using Microsoft.Syslog.Model; 13 | 14 | public static class ParserContextExtensions 15 | { 16 | public static bool ReadSymbol(this ParserContext ctx, char symbol, bool throwIfMismatch = true) 17 | { 18 | if (ctx.Current == SyslogChars.Space) 19 | { 20 | ctx.SkipSpaces(); 21 | } 22 | if (ctx.Current == symbol) 23 | { 24 | ctx.Position++; 25 | return true; 26 | } 27 | if (throwIfMismatch) 28 | throw new Exception($"Invalid input, expected '{symbol}' "); 29 | else 30 | return false; 31 | } 32 | 33 | public static bool TryReadUntil(this ParserContext ctx, string symbol, out string text) 34 | { 35 | var spos = ctx.Text.IndexOf(symbol, ctx.Position); 36 | if (spos > 0) 37 | { 38 | text = ctx.Text.Substring(ctx.Position, spos - ctx.Position); 39 | ctx.Position = spos; 40 | return true; 41 | } 42 | text = null; 43 | return false; 44 | } 45 | 46 | public static void SkipSpaces(this ParserContext ctx) 47 | { 48 | while (!ctx.Eof() && ctx.Current == SyslogChars.Space) 49 | ctx.Position++; 50 | } 51 | 52 | public static string ReadWordOrNil(this ParserContext ctx) 53 | { 54 | var word = ctx.ReadWord(); 55 | return (word == SyslogChars.Nil) ? null : word; 56 | } 57 | 58 | 59 | public static string ReadWord(this ParserContext ctx) 60 | { 61 | if (ctx.Current == SyslogChars.Space) 62 | { 63 | ctx.SkipSpaces(); 64 | } 65 | 66 | var separatorPos = ctx.Text.IndexOfAny(SyslogChars.WordSeparators, ctx.Position); 67 | if (separatorPos < 0) 68 | { 69 | separatorPos = ctx.Text.Length; 70 | } 71 | 72 | var word = ctx.Text.Substring(ctx.Position, separatorPos - ctx.Position); 73 | ctx.Position = separatorPos; 74 | return word; 75 | } 76 | 77 | public static string ReadQuotedString(this ParserContext ctx) 78 | { 79 | ctx.ReadSymbol(SyslogChars.DQuote); 80 | string result = string.Empty; 81 | int curr = ctx.Position; 82 | while (true) 83 | { 84 | var next = ctx.Text.IndexOfAny(SyslogChars.QuoteOrEscape, curr); 85 | var segment = ctx.Text.Substring(curr, next - curr); 86 | result += segment; 87 | switch (ctx.CharAt(next)) 88 | { 89 | case SyslogChars.DQuote: 90 | // we are done 91 | ctx.Position = next + 1; // after dquote 92 | return result; 93 | 94 | case SyslogChars.Escape: 95 | // it is escape symbol, add next char to result, shift to next char and continue loop 96 | result += ctx.CharAt(next + 1); 97 | curr = next + 2; 98 | break; 99 | }//switch 100 | } // loop 101 | } 102 | 103 | public static int ReadNumber(this ParserContext ctx, int maxDigits = 10) 104 | { 105 | var digits = ctx.ReadDigits(maxDigits); 106 | return int.Parse(digits); 107 | 108 | } 109 | 110 | public static string ReadDigits(this ParserContext ctx, int maxDigits = 10) 111 | { 112 | var start = ctx.Position; 113 | for (int i = 0; i < maxDigits; i++) { 114 | if (!char.IsDigit(ctx.Current)) 115 | break; 116 | ctx.Position++; 117 | if (ctx.Position >= ctx.Text.Length) 118 | break; 119 | } 120 | if (ctx.Position == start) 121 | return null; 122 | var res = ctx.Text.Substring(start, ctx.Position - start); 123 | return res; 124 | } 125 | 126 | 127 | /// Parser standard (for all levels) prefix <n>. 128 | /// parser context. 129 | /// True if prefix read correctly; otherwise, false. 130 | public static bool ReadSyslogPrefix(this ParserContext ctx) 131 | { 132 | try 133 | { 134 | ctx.Position = 0; 135 | ctx.ReadSymbol(SyslogChars.LT); 136 | var digits = ctx.ReadDigits(3); 137 | ctx.ReadSymbol(SyslogChars.GT); 138 | ctx.Prefix = ctx.Text.Substring(0, ctx.Position); 139 | return true; 140 | } 141 | catch (Exception) 142 | { 143 | ctx.Position = 0; 144 | return false; 145 | } 146 | } 147 | 148 | public static void AssignFacilitySeverity(this ParserContext ctx) 149 | { 150 | 151 | var priStr = ctx.Prefix.Replace("<", string.Empty).Replace(">", string.Empty).Replace("?", string.Empty); 152 | if (!int.TryParse(priStr, out var pri)) 153 | { 154 | ctx.AddError($"Invalid priiority value '{priStr}', expected '' where ? is int."); 155 | return; 156 | } 157 | 158 | // parse priority -> facility + severity 159 | var intFacility = pri / 8; 160 | var intSeverity = pri % 8; 161 | ctx.Entry.Facility = (Facility)intFacility; 162 | ctx.Entry.Severity = (Severity)intSeverity; 163 | } 164 | 165 | public static bool Reset(this ParserContext ctx) 166 | { 167 | if (ctx.Prefix == null && !ctx.ReadSyslogPrefix()) 168 | { 169 | return false; 170 | } 171 | ctx.Position = ctx.Prefix.Length; 172 | return true; 173 | } 174 | 175 | public static bool Match(this ParserContext ctx, string token, 176 | StringComparison comparison = StringComparison.OrdinalIgnoreCase) 177 | { 178 | if (ctx.Position + token.Length > ctx.Text.Length) 179 | return false; 180 | var str = ctx.Text.Substring(ctx.Position, token.Length); 181 | if(str.Equals(token, comparison)) 182 | { 183 | ctx.Position += token.Length; 184 | return true; 185 | } 186 | return false; 187 | } 188 | 189 | public static string GetValue(this IList prms, string name) 190 | { 191 | return prms.FirstOrDefault(p => p.Name == name)?.Value; 192 | } 193 | 194 | public static string CutOffBOM(this string msg) 195 | { 196 | if (msg.StartsWith(SyslogChars.BOM, StringComparison.Ordinal)) 197 | msg = msg.Substring(SyslogChars.BOM.Length); 198 | return msg; 199 | } 200 | 201 | } //class 202 | } 203 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/Parsers/ISyslogMessageParser.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | /// General interface for a parser for specific variant/version of syslog. 10 | /// 11 | /// The top SyslogParser calls all registered variant parsers asking to parse a message. 12 | /// If a variant parser recognizes its version and can parse it, it should do it and return true. 13 | /// 14 | public interface ISyslogMessageParser 15 | { 16 | bool TryParse(ParserContext context); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/Parsers/KeyValueListParser.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using System; 10 | using System.Collections.Generic; 11 | using Microsoft.Syslog.Model; 12 | 13 | /// Parses list of key-value pairs; like Sophos firewal format: 14 | /// https://docs.sophos.com/nsg/sophos-firewall/v16058/Help/en-us/webhelp/onlinehelp/index.html#page/onlinehelp/WAFLogs.html 15 | /// 16 | class KeyValueListParser : ISyslogMessageParser 17 | { 18 | public bool TryParse(ParserContext ctx) 19 | { 20 | if (ctx.Current == SyslogChars.Space) 21 | ctx.SkipSpaces(); 22 | 23 | // typically entries start with 'device=' or 'date=' 24 | var match = ctx.Match("device=") || ctx.Match("date="); 25 | if (!match) 26 | { 27 | match = TryMatchAnyKey(ctx); 28 | } 29 | if (!match) 30 | return false; 31 | 32 | // It is the format for this parser 33 | ctx.Reset(); // Match(...) moved the position, so return to the start 34 | ctx.Entry.PayloadType = PayloadType.KeyValuePairs; 35 | var kvList = ReadKeyValuePairs(ctx); 36 | ctx.Entry.ExtractedData.AddRange(kvList); 37 | // try some known values and put them in the header 38 | var hdr = ctx.Entry.Header; 39 | hdr.HostName = kvList.GetValue("device_id"); 40 | var date = kvList.GetValue("date"); 41 | var time = kvList.GetValue("time"); 42 | if (date != null) 43 | { 44 | var dateTimeStr = $"{date}T{time}"; 45 | if (DateTime.TryParse(dateTimeStr, out var dt )) 46 | { 47 | hdr.Timestamp = dt; 48 | } 49 | } 50 | 51 | return true; 52 | } //method 53 | 54 | private List ReadKeyValuePairs(ParserContext ctx) 55 | { 56 | var prmList = new List(); 57 | NameValuePair lastPrm = null; 58 | /* 59 | 2 troubles here: 60 | */ 61 | while (!ctx.Eof()) 62 | { 63 | ctx.SkipSpaces(); 64 | var name = ctx.ReadWord(); 65 | if(!ctx.ReadSymbol('=', throwIfMismatch: false)) 66 | { 67 | // Some entries are malformed: double quoted strings 68 | // the result is that we do not find '=' after closing the quote. So we just add the rest to a separate param and exit 69 | var text = ctx.Text.Substring(ctx.Position); 70 | prmList.Add(new NameValuePair() { Name = "Message", Value = text }); 71 | return prmList; 72 | } 73 | ctx.SkipSpaces(); 74 | string value; 75 | if (ctx.Current == SyslogChars.DQuote) 76 | { 77 | // For double-quoted values, some values are malformed - they contain nested d-quoted strings that are not escaped. 78 | value = ctx.ReadQuotedString(); 79 | } 80 | else 81 | { 82 | // Special case: non quoted empty values, ex: ' a= b=234 '; value of 'a' is Empty. We check the char after we read the value, 83 | // and if it is '=', we back off, set value to empty. 84 | var saveP = ctx.Position; 85 | value = ctx.ReadWord(); 86 | if (ctx.Current == '=') 87 | { 88 | ctx.Position = saveP; 89 | value = string.Empty; 90 | } 91 | } 92 | lastPrm = new NameValuePair() { Name = name, Value = value }; 93 | prmList.Add(lastPrm); 94 | } 95 | return prmList; 96 | } 97 | 98 | // let try to match any key, like <120> abc = def 99 | private bool TryMatchAnyKey(ParserContext ctx) 100 | { 101 | if (!char.IsLetter(ctx.Current)) 102 | return false; 103 | var savePos = ctx.Position; 104 | var word = ctx.ReadWord(); 105 | ctx.SkipSpaces(); 106 | var result = ctx.Match("="); 107 | ctx.Position = savePos; 108 | return result; 109 | 110 | } 111 | 112 | 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/Parsers/PlainTextParser.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using System; 10 | 11 | public class PlainTextParser : ISyslogMessageParser 12 | { 13 | public bool TryParse(ParserContext ctx) 14 | { 15 | ctx.SkipSpaces(); 16 | ctx.Entry.PayloadType = Model.PayloadType.PlainText; 17 | ctx.Entry.Message = ctx.Text.Substring(ctx.Position); 18 | ctx.Entry.Header.Timestamp = DateTime.UtcNow; 19 | return true; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/Parsers/Rfc3164SyslogParser.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | /// Parses old-style, BSD syslog, RFC-3164. See https://tools.ietf.org/html/rfc3164 10 | public class Rfc3164SyslogParser : ISyslogMessageParser 11 | { 12 | 13 | public bool TryParse(ParserContext ctx) 14 | { 15 | if (!TimestampParseHelper.TryParseTimestamp(ctx)) 16 | return false; 17 | 18 | var entry = ctx.Entry; 19 | entry.PayloadType = Model.PayloadType.Rfc3164; 20 | //Next - host name and proc name 21 | entry.Header.HostName = ctx.ReadWord(); 22 | ctx.SkipSpaces(); 23 | var procStart = ctx.Position; 24 | var procEnd = ctx.Text.SkipUntil(procStart + 1, ' ', ':', ','); 25 | if (procEnd < ctx.Text.Length) 26 | { 27 | var proc = ctx.Text.Substring(procStart, procEnd - procStart); 28 | entry.Header.ProcId = proc; 29 | ctx.Position = procEnd + 1; 30 | } 31 | if (ctx.Position < ctx.Text.Length) 32 | { 33 | // the rest is message 34 | ctx.SkipSpaces(); 35 | entry.Message = ctx.Text.Substring(ctx.Position); 36 | } 37 | return true; 38 | } 39 | 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/Parsers/Rfc5424SyslogParser.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using System; 10 | using System.Collections.Generic; 11 | using Microsoft.Syslog.Model; 12 | 13 | public class Rfc5424SyslogParser: ISyslogMessageParser 14 | { 15 | public bool TryParse(ParserContext ctx) 16 | { 17 | if (!ctx.Reset()) 18 | return false; 19 | if(!ctx.Match("1 ")) 20 | { 21 | return false; 22 | } 23 | 24 | // It is RFC-5424 entry 25 | var entry = ctx.Entry; 26 | entry.PayloadType = PayloadType.Rfc5424; 27 | try 28 | { 29 | entry.Header = this.ParseHeader(ctx); 30 | this.ParseStructuredData(ctx); 31 | entry.Message = this.ParseMessage(ctx); 32 | return true; 33 | } 34 | catch (Exception ex) 35 | { 36 | ctx.AddError(ex.Message); 37 | return false; 38 | } 39 | } 40 | 41 | private SyslogHeader ParseHeader(ParserContext ctx) 42 | { 43 | var header = ctx.Entry.Header = new SyslogHeader(); 44 | header.Timestamp = ctx.ParseStandardTimestamp(); 45 | header.HostName = ctx.ReadWordOrNil(); 46 | header.AppName = ctx.ReadWordOrNil(); 47 | header.ProcId = ctx.ReadWordOrNil(); 48 | header.MsgId = ctx.ReadWordOrNil(); 49 | return header; 50 | } 51 | 52 | private void ParseStructuredData(ParserContext ctx) 53 | { 54 | ctx.SkipSpaces(); 55 | 56 | if (ctx.Current == SyslogChars.NilChar) 57 | { 58 | ctx.Position++; 59 | return; 60 | } 61 | 62 | var data = ctx.Entry.StructuredData; 63 | try 64 | { 65 | if (ctx.Current != SyslogChars.Lbr) 66 | { 67 | // do not report it as an error, some messages out there are a bit malformed 68 | // ctx.AddError("Expected [ for structured data."); 69 | return; 70 | } 71 | // start parsing elements 72 | while(!ctx.Eof()) 73 | { 74 | var elem = ParseElement(ctx); 75 | if (elem == null) 76 | { 77 | return; 78 | } 79 | data[elem.Item1] = elem.Item2; 80 | } 81 | 82 | } catch (Exception ex) 83 | { 84 | ctx.AddError(ex.Message); 85 | } 86 | } 87 | 88 | private Tuple> ParseElement(ParserContext ctx) 89 | { 90 | if (ctx.Current != SyslogChars.Lbr) 91 | { 92 | return null; 93 | } 94 | ctx.Position++; 95 | var elemName = ctx.ReadWord(); 96 | ctx.SkipSpaces(); 97 | var paramList = new List(); 98 | var elem = new Tuple>(elemName, paramList); 99 | while (ctx.Current != SyslogChars.Rbr) 100 | { 101 | var paramName = ctx.ReadWord(); 102 | ctx.ReadSymbol('='); 103 | var paramValue = ctx.ReadQuotedString(); 104 | var prm = new NameValuePair() { Name = paramName, Value = paramValue }; 105 | paramList.Add(prm); 106 | ctx.SkipSpaces(); 107 | } 108 | 109 | ctx.ReadSymbol(SyslogChars.Rbr); 110 | return elem; 111 | } 112 | 113 | private string ParseMessage(ParserContext ctx) 114 | { 115 | if (ctx.Eof()) 116 | { 117 | return null; 118 | } 119 | var msg = ctx.Text.Substring(ctx.Position); 120 | msg = msg.TrimStart(SyslogChars.Space); 121 | // RFC 5424 allows BOM (byte order mark, 3 byte sequence) to precede the actual message. 122 | // it will be read into the message OK, now 'msg' can contain this prefix - it is invisible 123 | // and will bring a lot of trouble when working with the string (ex: string comparisons are broken) 124 | // So we remove it explicitly. 125 | return msg.CutOffBOM(); 126 | } 127 | 128 | 129 | } //class 130 | 131 | } 132 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/StringExtensions.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using System; 10 | using System.Linq; 11 | 12 | public static class StringExtensions 13 | { 14 | public static bool IsIpV4Char(this char ch) 15 | { 16 | return ch == SyslogChars.Dot || char.IsDigit(ch); 17 | } 18 | 19 | public static bool IsHexDigit(this char ch) 20 | { 21 | return char.IsDigit(ch) || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F'); 22 | } 23 | 24 | public static bool IsIpV6Char(this char ch) 25 | { 26 | return ch == SyslogChars.Colon || IsHexDigit(ch); 27 | } 28 | 29 | public static int SkipUntil(this string message, int start, Func func) 30 | { 31 | var p = start; 32 | while (p < message.Length && !func(message[p])) 33 | p++; 34 | return p; 35 | } 36 | 37 | public static int Skip(this string message, int start, params char[] chars) 38 | { 39 | var p = start; 40 | while (p < message.Length && chars.Contains(message[p])) 41 | p++; 42 | return p; 43 | } 44 | public static int SkipUntil(this string message, int start, params char[] chars) 45 | { 46 | var p = start; 47 | while (p < message.Length && !chars.Contains(message[p])) 48 | p++; 49 | return p; 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/SyslogChars.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using System.Text; 10 | 11 | public static class SyslogChars 12 | { 13 | public const string Nil = "-"; 14 | public const char NilChar = '-'; 15 | public const char Space = ' '; 16 | public const char LT = '<'; 17 | public const char GT = '>'; 18 | public const char Lbr = '['; 19 | public const char Rbr = ']'; 20 | public const char Escape = '\\'; 21 | public const char DQuote = '"'; 22 | public const char Dot = '.'; 23 | public const char Colon = ':'; 24 | public const char EQ = '='; 25 | public static readonly char[] QuoteOrEscape = new char[] { DQuote, Escape }; 26 | public static readonly char[] WordSeparators = new char[] { Space, Lbr, Rbr, EQ }; 27 | 28 | // BOM - byte-order-mark, special byte sequence often used as string prefix to indicate unicode. 29 | // we generally strip it out whenever we find it, it breaks string operations if left in string values 30 | public static readonly string BOM = Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble()); 31 | 32 | public static readonly string SyslogStart = LT.ToString(); 33 | public static readonly string SyslogStartBom = BOM + LT; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/Parsing/SyslogParser.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog.Parsing 8 | { 9 | using Microsoft.Syslog.Model; 10 | using System; 11 | using System.Collections.Generic; 12 | using System.Linq; 13 | 14 | /// 15 | /// Configurable syslog message parser. 16 | /// 17 | /// 18 | /// Holds configurable lists of version parsers (for handling specific version/format) and 19 | /// list of value extractors for pattern-based extraction. 20 | /// 21 | public class SyslogParser 22 | { 23 | public IList VersionParsers => _versionParsers; 24 | public IList ValueExtractors => _valueExtractors; 25 | 26 | private readonly List _versionParsers = new List(); 27 | private readonly List _valueExtractors = new List(); 28 | 29 | /// Creates and configures a default parser, with support for all major syslog versions 30 | /// and IP addresses extractor. 31 | /// 32 | /// 33 | public static SyslogParser CreateDefault() 34 | { 35 | var parser = new SyslogParser(); 36 | parser.AddVersionParsers(new Rfc5424SyslogParser(), new KeyValueListParser(), 37 | new Rfc3164SyslogParser(), new PlainTextParser()); 38 | parser.AddValueExtractors(new IpAddressesExtractor()); 39 | return parser; 40 | } 41 | 42 | public void AddVersionParsers(params ISyslogMessageParser[] parsers) 43 | { 44 | _versionParsers.AddRange(parsers); 45 | } 46 | 47 | public void AddValueExtractors(params IValuesExtractor[] extractors) 48 | { 49 | _valueExtractors.AddRange(extractors); 50 | } 51 | 52 | public SyslogEntry Parse(string text) 53 | { 54 | var ctx = new ParserContext(text); 55 | TryParse(ctx); 56 | ctx.Entry.BuildAllDataDictionary(); // put all parsed/extracted data into AllData dictionary 57 | return ctx.Entry; 58 | } 59 | 60 | 61 | public bool TryParse(ParserContext context) 62 | { 63 | if (!context.ReadSyslogPrefix()) 64 | return false; 65 | context.Entry = new SyslogEntry(); 66 | context.AssignFacilitySeverity(); 67 | 68 | foreach (var parser in _versionParsers) 69 | { 70 | context.Reset(); 71 | try 72 | { 73 | if (parser.TryParse(context)) 74 | { 75 | ExtractDataFromMessage(context); 76 | return context.ErrorMessages.Count == 0; 77 | } 78 | } 79 | catch (Exception ex) 80 | { 81 | context.ErrorMessages.Add(ex.ToString()); 82 | ex.Data["SyslogMessage"] = context.Text; 83 | throw; 84 | } 85 | } 86 | return false; 87 | } 88 | 89 | private void ExtractDataFromMessage(ParserContext ctx) 90 | { 91 | var entry = ctx.Entry; 92 | // For RFC-5424 and KeyValue payload types, everything is already structured and extracted 93 | // But we want to run IP values detector against all of them 94 | switch (entry.PayloadType) 95 | { 96 | case PayloadType.Rfc5424: 97 | var allParams = entry.StructuredData.SelectMany(e => e.Value).ToList(); 98 | var Ips = IpAddressesExtractor.ExtractIpAddresses(allParams); 99 | entry.ExtractedData.AddRange(Ips); 100 | return; 101 | 102 | case PayloadType.KeyValuePairs: 103 | var Ips2 = IpAddressesExtractor.ExtractIpAddresses(entry.ExtractedData); 104 | entry.ExtractedData.AddRange(Ips2); 105 | return; 106 | } 107 | 108 | // otherwise run extractors from plain message 109 | if (entry == null || string.IsNullOrWhiteSpace(entry.Message) || entry.Message.Length < 10) 110 | return; 111 | 112 | var allValues = entry.ExtractedData; 113 | foreach(var extr in _valueExtractors) 114 | { 115 | var values = extr.ExtractValues(ctx); 116 | if (values != null && values.Count > 0) 117 | { 118 | allValues.AddRange(values); 119 | } 120 | } 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/ReadMe.md: -------------------------------------------------------------------------------- 1 | # Microsoft.Syslog package 2 | 3 | The package provides components for implementing [Syslog](https://en.wikipedia.org/wiki/Syslog) facility, both client (producer) and server (listener) parts. The implementation follows the guidelines from the [RFC-5424](https://tools.ietf.org/html/rfc5424) document. 4 | 5 | Major components: 6 | * **Syslog entry** - a set of classes to represent a syslog payload as a strongly typed object, with nested properties and elements. 7 | * **Syslog parser and serializer** - facilities to convert *SyslogEntry* object to string payload and vice versa. 8 | * **Syslog client** - sends syslog entries as UDP packets over the network to the target IP/port. 9 | * **Syslog listener (server)** - listens to syslog port (514); recieves, reads and parses the syslog messages; broadcasts the syslog entries through IObservable interface -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/SyslogClient.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog 8 | { 9 | using System.Net; 10 | using System.Net.Sockets; 11 | using System.Text; 12 | using Microsoft.Syslog.Model; 13 | 14 | public class SyslogClient 15 | { 16 | IPEndPoint _target; 17 | UdpClient _udpClient; 18 | public UdpClient Client => _udpClient; 19 | 20 | public SyslogClient(IPEndPoint target) 21 | { 22 | _target = target; 23 | _udpClient = new UdpClient(); 24 | } 25 | 26 | public SyslogClient(string ipAddress, int port = 514) 27 | { 28 | var addr = IPAddress.Parse(ipAddress); 29 | _target = new IPEndPoint(addr, port); 30 | _udpClient = new UdpClient(); 31 | } 32 | 33 | public void Send(SyslogEntry entry) 34 | { 35 | var payload = SyslogSerializer.Serialize(entry); 36 | Send(payload); 37 | } 38 | 39 | //can be used to send non-stanard entries 40 | public void Send(string payload) 41 | { 42 | var dgram = Encoding.UTF8.GetBytes(payload); 43 | _udpClient.Send(dgram, dgram.Length, _target); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/SyslogEventArgs.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog 8 | { 9 | using System; 10 | using Microsoft.Syslog.Model; 11 | 12 | public class SyslogEntryEventArgs : EventArgs 13 | { 14 | public ServerSyslogEntry ServerEntry { get; } 15 | 16 | internal SyslogEntryEventArgs(ServerSyslogEntry entry) 17 | { 18 | ServerEntry = entry; 19 | } 20 | } 21 | 22 | 23 | public class SyslogErrorEventArgs: EventArgs 24 | { 25 | public Exception Error { get; } 26 | 27 | internal SyslogErrorEventArgs(Exception error) 28 | { 29 | Error = error; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/SyslogSerializer.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog 8 | { 9 | using Microsoft.Syslog.Model; 10 | using Microsoft.Syslog.Parsing; 11 | using System.Collections.Generic; 12 | using System.Text; 13 | 14 | public static class SyslogSerializer 15 | { 16 | private const char Space = ' '; 17 | private const char NilChar = '-'; 18 | private const char Lbr = '['; 19 | private const char Rbr = ']'; 20 | private const char Escape = '\\'; 21 | private const char DQuote = '"'; 22 | 23 | static char[] _charsToEscape = new char[] { DQuote, Escape, Rbr }; //according to RFC-5424 these must be escaped 24 | 25 | public static string TimestampFormat = "yyyy'-'MM'-'ddTHH:mm:ss.fffZ"; 26 | 27 | public static string Serialize(SyslogEntry entry) 28 | { 29 | var writer = new StringBuilder(); 30 | writer.WriteHeader(entry); 31 | writer.WriteStructuredData(entry.StructuredData); 32 | if (!string.IsNullOrEmpty(entry.Message)) 33 | { 34 | writer.Append(SyslogChars.Space); 35 | writer.Append(entry.Message); 36 | } 37 | return writer.ToString(); 38 | } 39 | 40 | private static void WriteHeader(this StringBuilder writer, SyslogEntry entry) 41 | { 42 | writer.Append('<'); 43 | var pri = GetPriority(entry); 44 | writer.Append(pri); 45 | writer.Append('>'); 46 | writer.Append('1'); 47 | writer.Append(Space); 48 | var header = entry.Header; 49 | var tsStr = header.Timestamp == null ? string.Empty : header.Timestamp.Value.ToString(TimestampFormat); 50 | writer.AppendWordOrNil(tsStr); 51 | writer.AppendWordOrNil(header.HostName); 52 | writer.AppendWordOrNil(header.AppName); 53 | writer.AppendWordOrNil(header.ProcId); 54 | writer.AppendWordOrNil(header.MsgId); 55 | } 56 | 57 | private static void AppendWordOrNil(this StringBuilder writer, string word) 58 | { 59 | if (string.IsNullOrWhiteSpace(word)) 60 | { 61 | writer.Append(NilChar); 62 | } 63 | else 64 | { 65 | writer.Append(word); 66 | } 67 | writer.Append(Space); 68 | } 69 | 70 | private static int GetPriority(this SyslogEntry entry) 71 | { 72 | return ((int)entry.Facility) * 8 + (int)entry.Severity; 73 | } 74 | 75 | private static void WriteStructuredData(this StringBuilder writer, IDictionary> structuredData) 76 | { 77 | if (structuredData == null || structuredData.Count == 0) 78 | { 79 | writer.Append(NilChar); 80 | writer.Append(Space); 81 | return; 82 | } 83 | 84 | foreach(var de in structuredData) 85 | { 86 | var elemName = de.Key; 87 | var paramList = de.Value; 88 | writer.Append(Lbr); 89 | writer.Append(elemName); 90 | foreach(var prm in paramList) 91 | { 92 | writer.Append(Space); 93 | writer.Append(prm.Name); 94 | writer.Append('='); 95 | var prmValue = EscapeParamValue(prm.Value); 96 | writer.Append(DQuote); 97 | writer.Append(prmValue); 98 | writer.Append(DQuote); 99 | } 100 | writer.Append(Rbr); 101 | } 102 | } 103 | 104 | private static string EscapeParamValue(string value) 105 | { 106 | if (string.IsNullOrEmpty(value)) 107 | return string.Empty; 108 | if(value.IndexOfAny(_charsToEscape) < 0) 109 | return value; 110 | // replace with escaped 111 | var escaped = value.Replace(@"\", @"\\").Replace(@"""", @"\""").Replace(@"]", @"\]"); 112 | return escaped; 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /Source/Microsoft.Syslog/UdpListener.cs: -------------------------------------------------------------------------------- 1 | // /******************************************************** 2 | // * * 3 | // * Copyright (C) Microsoft * 4 | // * * 5 | // ********************************************************/ 6 | 7 | namespace Microsoft.Syslog 8 | { 9 | using System; 10 | using System.Net; 11 | using System.Net.Sockets; 12 | using System.Threading; 13 | using Microsoft.Syslog.Internals; 14 | using Microsoft.Syslog.Model; 15 | 16 | public class UdpListener: Observable, IDisposable 17 | { 18 | public UdpClient PortListener { get; private set; } 19 | private bool disposeClientOnDispose; 20 | public event EventHandler Error; 21 | 22 | private Thread _thread; 23 | private bool _running; 24 | 25 | public UdpListener(IPAddress address = null, int port = 514, int bufferSize = 10 * 1024 * 1024) 26 | { 27 | address = address ?? IPAddress.Parse("127.0.0.1"); 28 | var endPoint = new IPEndPoint(address, port); 29 | PortListener = new UdpClient(endPoint); 30 | PortListener.Client.ReceiveBufferSize = bufferSize; 31 | disposeClientOnDispose = true; 32 | } 33 | 34 | public UdpListener(UdpClient udpClient) 35 | { 36 | PortListener = udpClient; 37 | disposeClientOnDispose = false; 38 | } 39 | 40 | public void Start() 41 | { 42 | if (_running) 43 | { 44 | return; 45 | } 46 | _running = true; 47 | 48 | // Important: we need real high-pri thread here, not pool thread from Task.Run() 49 | // Note: going with multiple threads here results in broken messages, the recieved message gets cut-off 50 | _thread = new Thread(RunListenerLoop); 51 | _thread.Priority = ThreadPriority.Highest; 52 | _thread.Start(); 53 | } 54 | 55 | public void Stop() 56 | { 57 | if (_running) 58 | { 59 | PortListener.Close(); 60 | } 61 | _running = false; 62 | Thread.Sleep(10); 63 | } 64 | 65 | private void RunListenerLoop() 66 | { 67 | try 68 | { 69 | var remoteIp = new IPEndPoint(IPAddress.Any, 0); 70 | while (_running) 71 | { 72 | var bytes = PortListener.Receive(ref remoteIp); 73 | var packet = new UdpPacket() { ReceivedOn = DateTime.UtcNow, SourceIpAddress = remoteIp.Address, Data = bytes }; 74 | Broadcast(packet); 75 | } 76 | } 77 | catch (Exception ex) 78 | { 79 | if (!_running) 80 | return; // it is closing socket 81 | OnError(ex); 82 | } 83 | } 84 | 85 | private void OnError(Exception error) 86 | { 87 | Error?.Invoke(this, new SyslogErrorEventArgs(error)); 88 | } 89 | 90 | public void Dispose() 91 | { 92 | if (disposeClientOnDispose && PortListener != null) 93 | { 94 | PortListener.Dispose(); 95 | } 96 | } 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /Source/RealTimeKql/CommandLineParsing/Argument.cs: -------------------------------------------------------------------------------- 1 | namespace RealTimeKql 2 | { 3 | public class Argument 4 | { 5 | public string FriendlyName { get; private set; } 6 | public string HelpText { get; private set; } 7 | public bool IsRequired { get; private set; } 8 | public string Value { get; set; } 9 | 10 | public Argument(string friendlyName, string helpText, bool isRequired=false) 11 | { 12 | FriendlyName = friendlyName; 13 | HelpText = helpText; 14 | IsRequired = isRequired; 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Source/RealTimeKql/CommandLineParsing/Option.cs: -------------------------------------------------------------------------------- 1 | namespace RealTimeKql 2 | { 3 | public class Option 4 | { 5 | public string LongName { get; private set; } 6 | public string ShortName { get; private set; } 7 | public string HelpText { get; private set; } 8 | public bool IsRequired { get; private set; } 9 | public bool IsFlag { get; private set; } 10 | public bool WasSet { get; set; } 11 | public string Value { get; set; } 12 | 13 | public Option( 14 | string longName, 15 | string shortName, 16 | string helpText, 17 | bool isRequired=false, 18 | bool isFlag=false) 19 | { 20 | LongName = longName; 21 | ShortName = shortName; 22 | HelpText = helpText; 23 | IsRequired = isRequired; 24 | IsFlag = isFlag; 25 | WasSet = false; 26 | } 27 | 28 | public bool IsEqual(string option) 29 | { 30 | var tmp = option.Trim('-'); 31 | var name = tmp.Split('=')[0]; 32 | return name == LongName || name == ShortName; 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Source/RealTimeKql/CommandLineParsing/Subcommand.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | using System.Linq; 3 | 4 | namespace RealTimeKql 5 | { 6 | public class Subcommand 7 | { 8 | public string Name { get; private set; } 9 | public string HelpText { get; private set; } 10 | public Argument Argument { get; private set; } 11 | public List