├── .github └── workflows │ ├── build.yml │ └── deploy.yml ├── .gitignore ├── .rubocop.yml ├── .vscode ├── settings.json └── tasks.json ├── CHANGELOG.md ├── CONTRIBUTORS ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── SECURITY.md ├── build.gradle ├── e2e ├── dataset.csv ├── dataset_mapping.json └── e2e.rb ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── lib └── logstash │ └── outputs │ ├── kusto.rb │ └── kusto │ ├── ingestor.rb │ └── interval.rb ├── logstash-output-kusto.gemspec ├── settings.gradle ├── spec ├── outputs │ ├── kusto │ │ └── ingestor_spec.rb │ └── kusto_spec.rb └── spec_helpers.rb └── version /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | 6 | jobs: 7 | build: 8 | name: Build gem 9 | runs-on: ubuntu-latest 10 | environment: build 11 | permissions: 12 | checks: write 13 | pull-requests: write 14 | id-token: write 15 | contents: read 16 | strategy: 17 | matrix: 18 | logstash: [ 19 | { version: '8.7.0', url: "https://artifacts.elastic.co/downloads/logstash/logstash-8.7.0-linux-x86_64.tar.gz" , main: 'true' } 20 | ] 21 | env: 22 | LOGSTASH_SOURCE: 1 23 | LOGSTASH_PATH: /home/runner/logstash 24 | JRUBY_HOME: /home/runner/logstash/vendor/jruby 25 | JAVA_HOME: /home/runner/logstash/jdk 26 | steps: 27 | - name: Azure login 28 | uses: azure/login@v2 29 | with: 30 | client-id: ${{ secrets.APP_ID }} 31 | tenant-id: ${{ secrets.AUTH_ID }} 32 | subscription-id: ${{ secrets.SUBSCRIPTION_ID }} 33 | - name: Build logstash 34 | run: | 35 | echo "Getting logstash version ${{matrix.logstash.version}}" 36 | wget -O logstash.tar.gz ${{matrix.logstash.url}} 37 | tar -xf logstash.tar.gz 38 | mv logstash*/ logstash 39 | working-directory: /home/runner 40 | - name: Set Path 41 | run: | 42 | echo "$LOGSTASH_PATH/bin" >> $GITHUB_PATH 43 | echo "$LOGSTASH_PATH/vendor/jruby/bin" >> $GITHUB_PATH 44 | - name: Install bundler 45 | run: jruby -S gem install bundler -v 2.4.19 46 | - name: Checkout code 47 | uses: actions/checkout@v2 48 | with: 49 | path: 'kusto' 50 | - name: List CWD 51 | run: ls -al 52 | working-directory: 'kusto' 53 | - run: jruby -S bundle install 54 | working-directory: 'kusto' 55 | - name: Run gradle vendor 56 | run: ./gradlew vendor 57 | working-directory: 'kusto' 58 | - name: Run unit tests 59 | run: jruby -S bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o rspec.xml 60 | working-directory: 'kusto' 61 | - run: jruby -S gem build *.gemspec 62 | working-directory: 'kusto' 63 | - run: mv *.gem logstash-kusto.gem 64 | working-directory: 'kusto' 65 | - if: matrix.logstash.main == 'true' 66 | name: Upload gem 67 | uses: actions/upload-artifact@v4 68 | with: 69 | name: logstash-kusto.gem 70 | path: 'kusto/logstash-kusto.gem' 71 | - if: matrix.logstash.main == 'true' 72 | name: Publish Unit Test Results 73 | uses: EnricoMi/publish-unit-test-result-action@v1.6 74 | with: 75 | github_token: ${{ secrets.GITHUB_TOKEN }} 76 | files: kusto/rspec.xml 77 | e2e: 78 | name: End-To-End Testing 79 | runs-on: ubuntu-latest 80 | environment: build 81 | permissions: 82 | checks: write 83 | pull-requests: write 84 | id-token: write 85 | contents: read 86 | needs: build 87 | steps: 88 | - name: Azure login 89 | uses: azure/login@v2 90 | with: 91 | client-id: ${{ secrets.APP_ID }} 92 | tenant-id: ${{ secrets.AUTH_ID }} 93 | subscription-id: ${{ secrets.SUBSCRIPTION_ID }} 94 | - uses: ruby/setup-ruby@v1 95 | with: 96 | ruby-version: jruby 97 | bundler-cache: true 98 | - name: Checkout code 99 | uses: actions/checkout@v2 100 | - name: Download gem 101 | uses: actions/download-artifact@v4 102 | with: 103 | name: logstash-kusto.gem 104 | - name: Install logstash # taken from logstash's website https://www.elastic.co/guide/en/logstash/7.10/installing-logstash.html#_apt 105 | run: | 106 | wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - 107 | sudo apt-get install apt-transport-https 108 | echo "deb https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-8.x.list 109 | sudo apt-get update && sudo apt-get install logstash 110 | - run: sudo chown -R runner /usr/share/logstash 111 | - run: sudo chmod -R u=rx,g=,o= /usr/share/logstash/bin 112 | - name: Install plugin 113 | run: /usr/share/logstash/bin/logstash-plugin install logstash-kusto.gem 114 | - name: Install libraries 115 | run: ./gradlew vendor 116 | - name: Run e2e 117 | run: jruby -J-cp ../vendor/jar-dependencies e2e.rb 118 | working-directory: 'e2e' 119 | env: 120 | ENGINE_URL: ${{ secrets.ENGINE_URL }} 121 | INGEST_URL: ${{ secrets.INGEST_URL }} 122 | TEST_DATABASE: ${{ secrets.TEST_DATABASE }} -------------------------------------------------------------------------------- /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_dispatch: 3 | release: 4 | types: [released] 5 | 6 | name: Deploy and Create Release 7 | 8 | jobs: 9 | build: 10 | permissions: 11 | checks: write 12 | pull-requests: write 13 | id-token: write 14 | packages: write # for pushing GitHub packages 15 | name: Upload Release Asset 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: ruby/setup-ruby@v1 19 | with: 20 | ruby-version: jruby 21 | bundler-cache: true 22 | - name: Checkout code 23 | uses: actions/checkout@v2 24 | - name: Run gradle vendor 25 | run: ./gradlew vendor 26 | - name: Build gem 27 | run: jruby -S gem build *.gemspec 28 | - name: Get release 29 | id: get_release 30 | uses: bruceadams/get-release@v1.2.2 31 | env: 32 | GITHUB_TOKEN: ${{ github.token }} 33 | - name: Publish gem 34 | uses: dawidd6/action-publish-gem@v1 35 | with: 36 | api_key: ${{secrets.RUBYGEMS_KEY}} 37 | github_token: ${{secrets.GITHUB_TOKEN}} 38 | - name: Set artifact name 39 | id: set_artifact_name 40 | run: | 41 | ARTIFACT_PATH=$(find . -maxdepth 1 -iname '*.gem') 42 | echo "::set-output name=artifact_name::$ARTIFACT_PATH" 43 | - name: Upload Release Asset 44 | id: upload-release-asset 45 | uses: actions/upload-release-asset@v1 46 | env: 47 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 48 | with: 49 | upload_url: ${{ steps.get_release.outputs.upload_url }} 50 | asset_path: ./${{ steps.set_artifact_name.outputs.artifact_name }} 51 | asset_name: ${{ steps.set_artifact_name.outputs.artifact_name }} 52 | asset_content_type: application/zip 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore bundler config 2 | /.bundle 3 | 4 | # Ignore the default SQLite database. 5 | /db/*.sqlite3 6 | 7 | # Ignore all logfiles and tempfiles. 8 | /log/*.log 9 | /tmp 10 | 11 | *.gem 12 | *.rbc 13 | .bundle 14 | .config 15 | coverage 16 | InstalledFiles 17 | lib/bundler/man 18 | pkg 19 | rdoc 20 | spec/reports 21 | test/tmp 22 | test/version_tmp 23 | tmp 24 | 25 | # this is a GEM project 26 | Gemfile.lock 27 | 28 | # VSCode stuff 29 | .vscode/* 30 | !.vscode/settings.json 31 | !.vscode/tasks.json 32 | !.vscode/launch.json 33 | !.vscode/extensions.json 34 | .vscode-server/* 35 | .ssh/* 36 | #Devcontainer 37 | *.devcontainer/* 38 | Jars.lock 39 | .mvn/* 40 | 41 | #logstash conf 42 | *.conf 43 | 44 | #IDEA 45 | .idea 46 | .idea/* 47 | 48 | #Jar files and generated files 49 | **/*.jar 50 | **/*_jars.rb 51 | # Ignore Gradle project-specific cache directory 52 | .gradle 53 | 54 | # Ignore Gradle build output directory 55 | build 56 | .vscode/settings.json 57 | gradle/wrapper/gradle-wrapper.properties 58 | .vscode/settings.json 59 | .vscode/settings.json 60 | rspec.xml 61 | e2e/output_file.txt 62 | logs.txt 63 | docker-e2e/.env 64 | local-run.sh 65 | logs2.txt 66 | **/.vscode/*.* 67 | **/settings.json 68 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | Metrics/LineLength: 2 | # Change the default 80 chars limit value 3 | Max: 120 4 | 5 | Lint/UselessAccessModifier: 6 | ContextCreatingMethods: 7 | - class_methods -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | // Basic settings: turn linter(s) on 3 | "ruby.lint": { 4 | "rubocop": true 5 | }, 6 | // Time (ms) to wait after keypress before running enabled linters. Ensures 7 | // linters are only run when typing has finished and not for every keypress 8 | "ruby.lintDebounceTime": 500, 9 | "ruby.format": "rubocop", 10 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "2.0.0", 5 | "tasks": [ 6 | { 7 | "label": "Build Gem", 8 | "type": "shell", 9 | "command": "gem build logstash-output-kusto.gemspec", 10 | "group": { 11 | "kind": "build", 12 | "isDefault": true 13 | }, 14 | "problemMatcher": [] 15 | }, 16 | { 17 | "label": "Run tests", 18 | "type": "shell", 19 | "command": "bundle exec rspec spec", 20 | "group": { 21 | "kind": "build", 22 | "isDefault": true 23 | } 24 | } 25 | ] 26 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 4 | # 2.0.3 5 | 6 | - Make JSON mapping optional 7 | 8 | 9 | # 2.0.2 10 | 11 | - Bugfix for the scenario where the plugin uses managed identity. Instead of providing the managed identity name as empty in the config, 12 | it can completely be skipped 13 | 14 | 15 | # 2.0.0 16 | 17 | - Use (5.0.2) version of the java sdk, and retrieve it from maven with bundler. Supports logstash 8.6 versions and up 18 | - Upgrade to latest Java SDK fixes [CVE](https://github.com/advisories/GHSA-599f-7c49-w659) and addresses Issue#48 19 | 20 | # 1.0.5 21 | 22 | - Use (3.1.3) version of the java sdk, and retrieve it from maven with bundler. 23 | - Added support for `proxy_host` `proxy_port` `proxy_protocol` to support proxying ingestion to Kusto 24 | 25 | # 1.0.0 26 | 27 | - Use stable (2.1.2) version of the java sdk, and retrieve it from maven with bundler. 28 | - Renamed `mapping` to `json_mapping` in order to clarify usage. `mapping` still remains as a deprecated parameter. 29 | 30 | ## 0.4.0 31 | 32 | - set 'client name for tracing' to identify usage of this plugin on Kusto logs 33 | 34 | ## 0.3.0 35 | 36 | - move to version 1.0.0-BETA-04 of azure-kusto-java sdk 37 | - better support multiple kusto outputs running in parallel 38 | 39 | ## 0.2.0 40 | 41 | - move to version 1.0.0-BETA-01 of azure-kusto-java sdk 42 | 43 | ## 0.1.7 44 | 45 | - fixed app_key (password) bug, include 0.1.7 of the kusto-java-sdk to allow working through a proxy 46 | 47 | ## 0.1.6 48 | 49 | - plugin published to the public. supports ingestion json events into a specific table-database (without dynamic routing currently) 50 | 51 | 52 | ## 0.1.0 53 | 54 | - Plugin created with the logstash plugin generator -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The following is a list of people who have contributed ideas, code, bug 2 | reports, or in general have helped logstash along its way. 3 | 4 | Contributors: 5 | * Tamir Kamara - tamir.kamara@microsoft.com 6 | 7 | Note: If you've sent us patches, bug reports, or otherwise contributed to 8 | Logstash, and you aren't on the list above and want to be, please let us know 9 | and we'll make sure you're here. Contributions from folks like you are what make 10 | open source awesome. 11 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" 6 | use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" 7 | 8 | if Dir.exist?(logstash_path) && use_logstash_source 9 | gem 'logstash-core', :path => "#{logstash_path}/logstash-core" 10 | gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" 11 | end -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright Microsoft Corporation. All rights reserved. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Logstash Output Plugin for Azure Data Explorer (Kusto) 2 | 3 | ![build](https://github.com/Azure/logstash-output-kusto/workflows/build/badge.svg) 4 | ![build](https://github.com/Azure/logstash-output-kusto/workflows/build/badge.svg?branch=master) 5 | [![Gem](https://img.shields.io/gem/v/logstash-output-kusto.svg)](https://rubygems.org/gems/logstash-output-kusto) 6 | [![Gem](https://img.shields.io/gem/dt/logstash-output-kusto.svg)](https://rubygems.org/gems/logstash-output-kusto) 7 | 8 | This is a plugin for [Logstash](https://github.com/elastic/logstash). 9 | 10 | It is fully free and open source. The license is Apache 2.0. 11 | 12 | This Azure Data Explorer (ADX) Logstash plugin enables you to process events from Logstash into an **Azure Data Explorer** database for later analysis. 13 | 14 | This connector forwards data to 15 | [Azure Data Explorer](https://docs.microsoft.com/en-us/azure/data-explorer), 16 | [Azure Synapse Data Explorer](https://docs.microsoft.com/en-us/azure/synapse-analytics/data-explorer/data-explorer-overview) and 17 | [Real time analytics in Fabric](https://learn.microsoft.com/en-us/fabric/real-time-analytics/overview) 18 | 19 | ## Requirements 20 | 21 | - Logstash version 6+. [Installation instructions](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) 22 | - Azure Data Explorer cluster with a database. Read [Create a cluster and database](https://docs.microsoft.com/en-us/azure/data-explorer/create-cluster-database-portal) for more information. 23 | - AAD Application credentials with permission to ingest data into Azure Data Explorer. Read [Creating an AAD Application](https://docs.microsoft.com/en-us/azure/kusto/management/access-control/how-to-provision-aad-app) for more information. 24 | 25 | ## Installation 26 | 27 | To make the Azure Data Explorer plugin available in your Logstash environment, run the following command: 28 | ```sh 29 | bin/logstash-plugin install logstash-output-kusto 30 | ``` 31 | 32 | ## Configuration 33 | 34 | Perform configuration before sending events from Logstash to Azure Data Explorer. The following example shows the minimum you need to provide. It should be enough for most use-cases: 35 | 36 | ```ruby 37 | output { 38 | kusto { 39 | path => "/tmp/kusto/%{+YYYY-MM-dd-HH-mm}.txt" 40 | ingest_url => "https://ingest-.kusto.windows.net/" 41 | app_id => "" 42 | app_key => "" 43 | app_tenant => "" 44 | database => "" 45 | table => "" 46 | json_mapping => "" 47 | proxy_host => "" 48 | proxy_port => 49 | proxy_protocol => <"http"|"https"> 50 | } 51 | } 52 | ``` 53 | More information about configuring Logstash can be found in the [logstash configuration guide](https://www.elastic.co/guide/en/logstash/current/configuration.html) 54 | 55 | ### Available Configuration Keys 56 | 57 | | Parameter Name | Description | Notes | 58 | | --- | --- | --- | 59 | | **path** | The plugin writes events to temporary files before sending them to ADX. This parameter includes a path where files should be written and a time expression for file rotation to trigger an upload to the ADX service. The example above shows how to rotate the files every minute and check the Logstash docs for more information on time expressions. | Required 60 | | **ingest_url** | The Kusto endpoint for ingestion-related communication. See it on the Azure Portal.| Required| 61 | | **app_id, app_key, app_tenant**| Credentials required to connect to the ADX service. Be sure to use an application with 'ingest' privileges. | Optional| 62 | | **managed_identity**| Managed Identity to authenticate. For user-based managed ID, use the Client ID GUID. For system-based, use the value `system`. The ID needs to have 'ingest' privileges on the cluster. | Optional| 63 | | **database**| Database name to place events | Required | 64 | | **table** | Target table name to place events | Required 65 | | **json_mapping** | Maps each attribute from incoming event JSON strings to the appropriate column in the table. Note that this must be in JSON format, as this is the interface between Logstash and Kusto | Optional | 66 | | **recovery** | If set to true (default), plugin will attempt to resend pre-existing temp files found in the path upon startup | | 67 | | **delete_temp_files** | Determines if temp files will be deleted after a successful upload (true is default; set false for debug purposes only)| | 68 | | **flush_interval** | The time (in seconds) for flushing writes to temporary files. Default is 2 seconds, 0 will flush on every event. Increase this value to reduce IO calls but keep in mind that events in the buffer will be lost in case of abrupt failure.| | 69 | | **proxy_host** | The proxy hostname for redirecting traffic to Kusto.| | 70 | | **proxy_port** | The proxy port for the proxy. Defaults to 80.| | 71 | | **proxy_protocol** | The proxy server protocol , is one of http or https.| | 72 | 73 | > Note : LS_JAVA_OPTS can be used to set proxy parameters as well (using export or SET options) 74 | 75 | ```bash 76 | export LS_JAVA_OPTS="-Dhttp.proxyHost=1.2.34 -Dhttp.proxyPort=8989 -Dhttps.proxyHost=1.2.3.4 -Dhttps.proxyPort=8989" 77 | ``` 78 | 79 | 80 | ### Release Notes and versions 81 | 82 | | Version | Release Date | Notes | 83 | | --- | --- | --- | 84 | | 2.0.8 | 2024-10-23 | - Fix library deprecations, fix issues in the Azure Identity library | 85 | | 2.0.7 | 2024-01-01 | - Update Kusto JAVA SDK | 86 | | 2.0.3 | 2023-12-12 | - Make JSON mapping field optional. If not provided logstash output JSON attribute names will be used for column resolution | 87 | | 2.0.2 | 2023-11-28 | - Bugfix for the scenario where the plugin uses managed identity. Instead of providing the managed identity name as empty in the config,it can completely be skipped | 88 | | 2.0.0 | 2023-09-19 | - Upgrade to the latest Java SDK version [5.0.2](https://github.com/Azure/azure-kusto-java/releases/tag/v5.0.2). Tests have been performed on **__Logstash 8.5__** and up (Does not work with 6.x or 7.x versions of Logstash - For these versions use 1.x.x versions of logstash-output-kusto gem) - Fixes CVE's in common-text & outdated Jackson libraries | 89 | | 1.0.6 | 2022-11-29 | - Upgrade to the latest Java SDK [3.2.1](https://github.com/Azure/azure-kusto-java/releases/tag/v3.2.1) version. Tests have been performed on Logstash 6.x and up.| 90 | 91 | 92 | ## Development Requirements 93 | 94 | - Openjdk **8 64bit** (https://www.openlogic.com/openjdk-downloads) 95 | - JRuby 9.2 or higher, defined with openjdk 8 64bit 96 | - Logstash, defined with openjdk 8 64bit 97 | 98 | *It is reccomened to use the bundled jdk and jruby with logstash to avoid compatibility issues.* 99 | 100 | To fully build the gem, run: 101 | 102 | ```shell 103 | bundle install 104 | lock_jars 105 | gem build 106 | ``` 107 | 108 | ## Contributing 109 | 110 | All contributions are welcome: ideas, patches, documentation, bug reports, and complaints. 111 | Programming is not a required skill. It is more important to the community that you are able to contribute. 112 | For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file. 113 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "logstash/devutils/rake" 3 | require "jars/installer" 4 | require "fileutils" 5 | 6 | task :default do 7 | system('rake -vT') 8 | end 9 | 10 | task :vendor do 11 | exit(1) unless system './gradlew vendor' 12 | end 13 | 14 | task :clean do 15 | ["vendor/jar-dependencies", "Gemfile.lock"].each do |p| 16 | FileUtils.rm_rf(p) 17 | end 18 | end -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | import java.nio.file.Files 2 | import java.nio.file.Paths 3 | 4 | import static java.nio.file.StandardCopyOption.REPLACE_EXISTING 5 | 6 | apply plugin: 'java' 7 | apply plugin: 'idea' 8 | apply plugin: 'maven-publish' 9 | 10 | // The gemspec contains the gem metadata to build and package the gem. The gradle build serves as a mechanism of getting these "vendor" files required for the gem. 11 | // The alternative is to use ruby-maven gem to package, but this runs into classpath conflicts/issues with the logstash plugin. 12 | group "org.logstash.outputs" 13 | 14 | def versionFile = Paths.get("version") 15 | if (Files.exists(versionFile)) { 16 | version = Files.readAllLines(versionFile).first() 17 | } else { 18 | version = "2.0.7" 19 | } 20 | 21 | repositories { 22 | mavenCentral() 23 | } 24 | 25 | // These dependencies are required by the gemspec to build the gem. The easiest to arrive at this list is to look at the effective pom of kusto-ingest and arrive at this list 26 | // even if we use the ruby-maven gem to package the gem, install and lock_jars will create the logstash_output_kusto_jars.rb file with the same list of dependencies. 27 | // In the gradle way, running ./gradlew vendor creates the jar file list and adds them to vendor/jar-dependencies folder from where it is referenced in the gemspec (require_paths and files) 28 | 29 | // update dependencies to bom azure-sdk-bom/1.2.24 30 | 31 | dependencies { 32 | implementation 'com.microsoft.azure.kusto:kusto-data:5.2.0' 33 | implementation 'com.microsoft.azure.kusto:kusto-ingest:5.2.0' 34 | implementation 'com.azure:azure-core-http-netty:1.15.1' 35 | implementation 'com.azure:azure-core:1.49.1' 36 | implementation 'com.azure:azure-data-tables:12.4.2' 37 | implementation 'com.azure:azure-identity:1.13.0' 38 | implementation 'com.azure:azure-json:1.1.0' 39 | implementation 'com.azure:azure-storage-blob:12.26.1' 40 | implementation 'com.azure:azure-storage-common:12.25.1' 41 | implementation 'com.azure:azure-storage-queue:12.21.1' 42 | implementation 'com.azure:azure-xml:1.0.0' 43 | implementation 'com.fasterxml.jackson.core:jackson-annotations:2.16.0' 44 | implementation 'com.fasterxml.jackson.core:jackson-core:2.16.0' 45 | implementation 'com.fasterxml.jackson.core:jackson-databind:2.16.0' 46 | implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.16.0' 47 | implementation 'com.fasterxml.woodstox:woodstox-core:6.7.0' 48 | implementation 'com.github.stephenc.jcip:jcip-annotations:1.0-1' 49 | implementation 'com.microsoft.azure:msal4j:1.15.1' 50 | implementation 'com.nimbusds:content-type:2.3' 51 | implementation 'com.nimbusds:lang-tag:1.7' 52 | implementation 'com.nimbusds:nimbus-jose-jwt:9.40' 53 | implementation 'com.nimbusds:oauth2-oidc-sdk:11.13' 54 | implementation 'com.univocity:univocity-parsers:2.9.1' 55 | implementation 'commons-codec:commons-codec:1.16.1' 56 | implementation 'commons-logging:commons-logging:1.3.1' 57 | implementation 'io.github.resilience4j:resilience4j-core:1.7.1' 58 | implementation 'io.github.resilience4j:resilience4j-retry:1.7.1' 59 | implementation 'io.netty:netty-buffer:4.1.118.Final' 60 | implementation 'io.netty:netty-codec-dns:4.1.118.Final' 61 | implementation 'io.netty:netty-codec-http2:4.1.118.Final' 62 | implementation 'io.netty:netty-codec-http:4.1.118.Final' 63 | implementation 'io.netty:netty-codec-socks:4.1.118.Final' 64 | implementation 'io.netty:netty-codec:4.1.118.Final' 65 | implementation 'io.netty:netty-common:4.1.118.Final' 66 | implementation 'io.netty:netty-handler-proxy:4.1.118.Final' 67 | implementation 'io.netty:netty-handler:4.1.118.Final' 68 | implementation 'io.netty:netty-resolver-dns-classes-macos:4.1.118.Final' 69 | implementation 'io.netty:netty-resolver-dns-native-macos:4.1.118.Final:osx-x86_64' 70 | implementation 'io.netty:netty-resolver-dns:4.1.118.Final' 71 | implementation 'io.netty:netty-resolver:4.1.118.Final' 72 | implementation 'io.netty:netty-tcnative-boringssl-static:2.0.65.Final' 73 | implementation 'io.netty:netty-tcnative-classes:2.0.65.Final' 74 | implementation 'io.netty:netty-transport-classes-epoll:4.1.118.Final' 75 | implementation 'io.netty:netty-transport-classes-kqueue:4.1.118.Final' 76 | implementation 'io.netty:netty-transport-native-epoll:4.1.118.Final:linux-x86_64' 77 | implementation 'io.netty:netty-transport-native-kqueue:4.1.118.Final:osx-x86_64' 78 | implementation 'io.netty:netty-transport-native-unix-common:4.1.118.Final' 79 | implementation 'io.netty:netty-transport:4.1.118.Final' 80 | implementation 'io.projectreactor.netty:reactor-netty-core:1.0.45' 81 | implementation 'io.projectreactor.netty:reactor-netty-http:1.0.45' 82 | implementation 'io.projectreactor:reactor-core:3.4.38' 83 | implementation 'io.vavr:vavr:0.10.4' 84 | implementation 'io.vavr:vavr-match:0.10.4' 85 | implementation 'net.java.dev.jna:jna-platform:5.13.0' 86 | implementation 'net.java.dev.jna:jna:5.13.0' 87 | implementation 'net.minidev:accessors-smart:2.5.2' 88 | implementation 'net.minidev:json-smart:2.5.2' 89 | implementation 'org.apache.commons:commons-lang3:3.14.0' 90 | implementation 'org.apache.commons:commons-text:1.11.0' 91 | implementation 'org.apache.httpcomponents:httpclient:4.5.14' 92 | implementation 'org.apache.httpcomponents:httpcore:4.4.16' 93 | implementation 'org.codehaus.woodstox:stax2-api:4.2.2' 94 | implementation 'org.jetbrains:annotations:24.1.0' 95 | implementation 'org.ow2.asm:asm:9.7' 96 | implementation 'org.reactivestreams:reactive-streams:1.0.4' 97 | implementation 'org.slf4j:slf4j-api:1.8.0-beta4' 98 | implementation 'org.slf4j:slf4j-simple:1.8.0-beta4' 99 | } 100 | 101 | // This task generates the ruby file with all the dependencies. Once this task runs, look at the lib/logstash-output-kusto_jars.rb file 102 | // that references all the dependent jar files. (the wrapper is the ./gradle vendor task) 103 | task generateGemJarRequiresFile { 104 | doLast { 105 | File jars_file = file('lib/logstash-output-kusto_jars.rb') 106 | jars_file.newWriter().withWriter { w -> 107 | w << "# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.\n\n" 108 | w << "require \'jar_dependencies\'\n" 109 | configurations.runtimeClasspath.allDependencies.each { 110 | w << "require_jar(\'${it.group}\', \'${it.name}\', \'${it.version}\')\n" 111 | } 112 | w << "require_jar(\'${project.group}\', \'${project.name}\', \'${project.version}\')\n" 113 | } 114 | } 115 | } 116 | // The vendor task copies the jars from the runtimeClasspath to the vendor/jar-dependencies folder. 117 | // This is referenced in the gemspec as the require_paths and files. Once this vendor task runs, look at the lib/logstash-output-kusto_jars.rb file 118 | // that references all the dependent jar files. 119 | task vendor { 120 | // take in all the dependencies from the runtimeClasspath and copy them to the vendor/jar-dependencies folder 121 | doLast { 122 | String vendorPathPrefix = "vendor/jar-dependencies" 123 | configurations.runtimeClasspath.allDependencies.each { dep -> 124 | println("Copying ${dep.group}:${dep.name}:${dep.version}") 125 | File f = configurations.runtimeClasspath.filter { it.absolutePath.contains("${dep.group}${File.separator}${dep.name}${File.separator}${dep.version}") }.singleFile 126 | String groupPath = dep.group.replaceAll('\\.', '/') 127 | File newJarFile = file("${vendorPathPrefix}${File.separator}${groupPath}${File.separator}${dep.name}${File.separator}${dep.version}${File.separator}${dep.name}-${dep.version}.jar") 128 | newJarFile.mkdirs() 129 | Files.copy(f.toPath(), newJarFile.toPath(), REPLACE_EXISTING) 130 | } 131 | String projectGroupPath = project.group.replaceAll('\\.', '/') 132 | File projectJarFile = file("${vendorPathPrefix}${File.separator}${projectGroupPath}${File.separator}${project.name}${File.separator}${project.version}${File.separator}${project.name}-${project.version}.jar") 133 | projectJarFile.mkdirs() 134 | Files.copy(file("$buildDir${File.separator}libs${File.separator}${project.name}-${project.version}.jar").toPath(), projectJarFile.toPath(), REPLACE_EXISTING) 135 | } 136 | } 137 | // The jar task is the standard packaging task & the generateGemJarRequiresFile task generates the ruby file with all the dependencies. 138 | vendor.dependsOn(jar, generateGemJarRequiresFile) -------------------------------------------------------------------------------- /e2e/dataset.csv: -------------------------------------------------------------------------------- 1 | 0,00000000-0000-0000-0001-020304050607,0.0,0.0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null 2 | 1,00000001-0000-0000-0001-020304050607,1.0001,1.01,1,1,1,1,1,1,1,1,2015-01-01T01:01:01.0000000Z,One,"One",1,00:00:01.0010001,,"{""arr"":[0,1],""rowId"":1}" 3 | 2,00000002-0000-0000-0001-020304050607,2.0002,2.02,0,2,2,2,2,2,2,2,2016-01-01T01:01:01.0000000Z,Two,"Two",2,-00:00:02.0020002,,"{""arr"":[0,2],""rowId"":2}" 4 | 3,00000003-0000-0000-0001-020304050607,3.0003,3.03,1,3,3,3,3,3,3,3,2017-01-01T01:01:01.0000000Z,Three,"Three",3,00:00:03.0030003,,"{""arr"":[0,3],""rowId"":3}" 5 | 4,00000004-0000-0000-0001-020304050607,4.0004,4.04,0,4,4,4,4,4,4,4,2018-01-01T01:01:01.0000000Z,Four,"Four",4,-00:00:04.0040004,,"{""arr"":[0,4],""rowId"":4}" 6 | 5,00000005-0000-0000-0001-020304050607,5.0005,5.05,1,5,5,5,5,5,5,5,2019-01-01T01:01:01.0000000Z,Five,"Five",5,00:00:05.0050005,,"{""arr"":[0,5],""rowId"":5}" 7 | 6,00000006-0000-0000-0001-020304050607,6.0006,6.06,0,6,6,6,6,6,6,6,2020-01-01T01:01:01.0000000Z,Six,"Six",6,-00:00:06.0060006,,"{""arr"":[0,6],""rowId"":6}" 8 | 7,00000007-0000-0000-0001-020304050607,7.0007,7.07,1,7,7,7,7,7,7,7,2021-01-01T01:01:01.0000000Z,Seven,"Seven",7,00:00:07.0070007,,"{""arr"":[0,7],""rowId"":7}" 9 | 8,00000008-0000-0000-0001-020304050607,8.0008,8.08,0,8,8,8,8,8,8,8,2022-01-01T01:01:01.0000000Z,Eight,"Eight",8,-00:00:08.0080008,,"{""arr"":[0,8],""rowId"":8}" 10 | 9,00000009-0000-0000-0001-020304050607,9.0009,9.09,1,9,9,9,9,9,9,9,2023-01-01T01:01:01.0000000Z,Nine,"Nine",9,00:00:09.0090009,,"{""arr"":[0,9],""rowId"":9}" 11 | -------------------------------------------------------------------------------- /e2e/dataset_mapping.json: -------------------------------------------------------------------------------- 1 | [{"Properties":{"Path":"$.rownumber"},"column":"rownumber","datatype":"int"},{"Properties":{"Path":"$.rowguid"},"column":"rowguid","datatype":"string"},{"Properties":{"Path":"$.xdouble"},"column":"xdouble","datatype":"real"},{"Properties":{"Path":"$.xfloat"},"column":"xfloat","datatype":"real"},{"Properties":{"Path":"$.xbool"},"column":"xbool","datatype":"bool"},{"Properties":{"Path":"$.xint16"},"column":"xint16","datatype":"int"},{"Properties":{"Path":"$.xint32"},"column":"xint32","datatype":"int"},{"Properties":{"Path":"$.xint64"},"column":"xint64","datatype":"long"},{"Properties":{"Path":"$.xuint8"},"column":"xuint8","datatype":"long"},{"Properties":{"Path":"$.xuint16"},"column":"xuint16","datatype":"long"},{"Properties":{"Path":"$.xuint32"},"column":"xuint32","datatype":"long"},{"Properties":{"Path":"$.xuint64"},"column":"xuint64","datatype":"long"},{"Properties":{"Path":"$.xdate"},"column":"xdate","datatype":"datetime"},{"Properties":{"Path":"$.xsmalltext"},"column":"xsmalltext","datatype":"string"},{"Properties":{"Path":"$.xtext"},"column":"xtext","datatype":"string"},{"Properties":{"Path":"$.rowguid"},"column":"xnumberAsText","datatype":"string"},{"Properties":{"Path":"$.xtime"},"column":"xtime","datatype":"timespan"},{"Properties":{"Path":"$.xtextWithNulls"},"column":"xtextWithNulls","datatype":"string"},{"Properties":{"Path":"$.xdynamicWithNulls"},"column":"xdynamicWithNulls","datatype":"dynamic"}] -------------------------------------------------------------------------------- /e2e/e2e.rb: -------------------------------------------------------------------------------- 1 | require '../lib/logstash-output-kusto_jars' 2 | require 'csv' 3 | 4 | $kusto_java = Java::com.microsoft.azure.kusto 5 | 6 | class E2E 7 | 8 | def initialize 9 | super 10 | @input_file = "/tmp/input_file.txt" 11 | @output_file = "output_file.txt" 12 | @columns = "(rownumber:int, rowguid:string, xdouble:real, xfloat:real, xbool:bool, xint16:int, xint32:int, xint64:long, xuint8:long, xuint16:long, xuint32:long, xuint64:long, xdate:datetime, xsmalltext:string, xtext:string, xnumberAsText:string, xtime:timespan, xtextWithNulls:string, xdynamicWithNulls:dynamic)" 13 | @csv_columns = '"rownumber", "rowguid", "xdouble", "xfloat", "xbool", "xint16", "xint32", "xint64", "xuint8", "xuint16", "xuint32", "xuint64", "xdate", "xsmalltext", "xtext", "xnumberAsText", "xtime", "xtextWithNulls", "xdynamicWithNulls"' 14 | @column_count = 19 15 | @engine_url = ENV["ENGINE_URL"] 16 | @ingest_url = ENV["INGEST_URL"] 17 | @database = ENV['TEST_DATABASE'] 18 | @lslocalpath = ENV['LS_LOCAL_PATH'] 19 | if @lslocalpath.nil? 20 | @lslocalpath = "/usr/share/logstash/bin/logstash" 21 | end 22 | @table_with_mapping = "RubyE2E#{Time.now.getutc.to_i}" 23 | @table_without_mapping = "RubyE2ENoMapping#{Time.now.getutc.to_i}" 24 | @mapping_name = "test_mapping" 25 | @csv_file = "dataset.csv" 26 | 27 | @logstash_config = %{ 28 | input { 29 | file { path => "#{@input_file}"} 30 | } 31 | filter { 32 | csv { columns => [#{@csv_columns}]} 33 | } 34 | output { 35 | file { path => "#{@output_file}"} 36 | stdout { codec => rubydebug } 37 | kusto { 38 | path => "tmp%{+YYYY-MM-dd-HH-mm}.txt" 39 | ingest_url => "#{@ingest_url}" 40 | cli_auth => true 41 | database => "#{@database}" 42 | table => "#{@table_with_mapping}" 43 | json_mapping => "#{@mapping_name}" 44 | } 45 | kusto { 46 | path => "nomaptmp%{+YYYY-MM-dd-HH-mm}.txt" 47 | cli_auth => true 48 | ingest_url => "#{@ingest_url}" 49 | database => "#{@database}" 50 | table => "#{@table_without_mapping}" 51 | } 52 | } 53 | } 54 | end 55 | 56 | def create_table_and_mapping 57 | Array[@table_with_mapping, @table_without_mapping].each { |tableop| 58 | puts "Creating table #{tableop}" 59 | @query_client.execute(@database, ".drop table #{tableop} ifexists") 60 | sleep(1) 61 | @query_client.execute(@database, ".create table #{tableop} #{@columns}") 62 | @query_client.execute(@database, ".alter table #{tableop} policy ingestionbatching @'{\"MaximumBatchingTimeSpan\":\"00:00:10\", \"MaximumNumberOfItems\": 1, \"MaximumRawDataSizeMB\": 100}'") 63 | } 64 | # Mapping only for one table 65 | @query_client.execute(@database, ".create table #{@table_with_mapping} ingestion json mapping '#{@mapping_name}' '#{File.read("dataset_mapping.json")}'") 66 | end 67 | 68 | 69 | def drop_and_cleanup 70 | Array[@table_with_mapping, @table_without_mapping].each { |tableop| 71 | puts "Dropping table #{tableop}" 72 | @query_client.execute(@database, ".drop table #{tableop} ifexists") 73 | sleep(1) 74 | } 75 | end 76 | 77 | def run_logstash 78 | File.write("logstash.conf", @logstash_config) 79 | logstashpath = File.absolute_path("logstash.conf") 80 | File.write(@output_file, "") 81 | File.write(@input_file, "") 82 | lscommand = "#{@lslocalpath} -f #{logstashpath}" 83 | puts "Running logstash from config path #{logstashpath} and final command #{lscommand}" 84 | spawn(lscommand) 85 | sleep(60) 86 | data = File.read(@csv_file) 87 | f = File.open(@input_file, "a") 88 | f.write(data) 89 | f.close 90 | sleep(60) 91 | puts File.read(@output_file) 92 | end 93 | 94 | def assert_data 95 | max_timeout = 10 96 | csv_data = CSV.read(@csv_file) 97 | Array[@table_with_mapping, @table_without_mapping].each { |tableop| 98 | puts "Validating results for table #{tableop}" 99 | (0...max_timeout).each do |_| 100 | begin 101 | sleep(5) 102 | query = @query_client.execute(@database, "#{tableop} | sort by rownumber asc") 103 | result = query.getPrimaryResults() 104 | raise "Wrong count - expected #{csv_data.length}, got #{result.count()} in table #{tableop}" unless result.count() == csv_data.length 105 | rescue Exception => e 106 | puts "Error: #{e}" 107 | end 108 | (0...csv_data.length).each do |i| 109 | result.next() 110 | puts "Item #{i}" 111 | (0...@column_count).each do |j| 112 | csv_item = csv_data[i][j] 113 | result_item = result.getObject(j) == nil ? "null" : result.getString(j) 114 | #special cases for data that is different in csv vs kusto 115 | if j == 4 #kusto boolean field 116 | csv_item = csv_item.to_s == "1" ? "true" : "false" 117 | elsif j == 12 # date formatting 118 | csv_item = csv_item.sub(".0000000", "") 119 | elsif j == 15 # numbers as text 120 | result_item = i.to_s 121 | elsif j == 17 #null 122 | next 123 | end 124 | puts " csv[#{j}] = #{csv_item}" 125 | puts " result[#{j}] = #{result_item}" 126 | raise "Result Doesn't match csv in table #{tableop}" unless csv_item == result_item 127 | end 128 | puts "" 129 | end 130 | return 131 | end 132 | raise "Failed after timeouts" 133 | } 134 | end 135 | 136 | def start 137 | @query_client = $kusto_java.data.ClientFactory.createClient($kusto_java.data.auth.ConnectionStringBuilder::createWithAzureCli(@engine_url)) 138 | create_table_and_mapping 139 | run_logstash 140 | assert_data 141 | drop_and_cleanup 142 | end 143 | end 144 | 145 | E2E::new().start -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/logstash-output-kusto/65113760263ed23ece702682011e2f94cbd07640/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Use "xargs" to parse quoted args. 209 | # 210 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 211 | # 212 | # In Bash we could simply go: 213 | # 214 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 215 | # set -- "${ARGS[@]}" "$@" 216 | # 217 | # but POSIX shell has neither arrays nor command substitution, so instead we 218 | # post-process each arg (as a line of input to sed) to backslash-escape any 219 | # character that might be a shell metacharacter, then use eval to reverse 220 | # that process (while maintaining the separation between arguments), and wrap 221 | # the whole thing up as a single "set" statement. 222 | # 223 | # This will of course break if any of these variables contains a newline or 224 | # an unmatched quote. 225 | # 226 | 227 | eval "set -- $( 228 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 229 | xargs -n1 | 230 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 231 | tr '\n' ' ' 232 | )" '"$@"' 233 | 234 | exec "$JAVACMD" "$@" 235 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /lib/logstash/outputs/kusto.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'logstash/outputs/base' 4 | require 'logstash/namespace' 5 | require 'logstash/errors' 6 | 7 | require 'logstash/outputs/kusto/ingestor' 8 | require 'logstash/outputs/kusto/interval' 9 | 10 | ## 11 | # This plugin sends messages to Azure Kusto in batches. 12 | # 13 | class LogStash::Outputs::Kusto < LogStash::Outputs::Base 14 | config_name 'kusto' 15 | concurrency :shared 16 | 17 | FIELD_REF = /%\{[^}]+\}/ 18 | 19 | attr_reader :failure_path 20 | 21 | # The path to the file to write. Event fields can be used here, 22 | # like `/var/log/logstash/%{host}/%{application}` 23 | # One may also utilize the path option for date-based log 24 | # rotation via the joda time format. This will use the event 25 | # timestamp. 26 | # E.g.: `path => "./test-%{+YYYY-MM-dd}.txt"` to create 27 | # `./test-2013-05-29.txt` 28 | # 29 | # If you use an absolute path you cannot start with a dynamic string. 30 | # E.g: `/%{myfield}/`, `/test-%{myfield}/` are not valid paths 31 | config :path, validate: :string, required: true 32 | 33 | # Flush interval (in seconds) for flushing writes to files. 34 | # 0 will flush on every message. Increase this value to recude IO calls but keep 35 | # in mind that events buffered before flush can be lost in case of abrupt failure. 36 | config :flush_interval, validate: :number, default: 2 37 | 38 | # If the generated path is invalid, the events will be saved 39 | # into this file and inside the defined path. 40 | config :filename_failure, validate: :string, default: '_filepath_failures' 41 | 42 | # If the configured file is deleted, but an event is handled by the plugin, 43 | # the plugin will recreate the file. Default => true 44 | config :create_if_deleted, validate: :boolean, default: true 45 | 46 | # Dir access mode to use. Note that due to the bug in jruby system umask 47 | # is ignored on linux: https://github.com/jruby/jruby/issues/3426 48 | # Setting it to -1 uses default OS value. 49 | # Example: `"dir_mode" => 0750` 50 | config :dir_mode, validate: :number, default: -1 51 | 52 | # File access mode to use. Note that due to the bug in jruby system umask 53 | # is ignored on linux: https://github.com/jruby/jruby/issues/3426 54 | # Setting it to -1 uses default OS value. 55 | # Example: `"file_mode" => 0640` 56 | config :file_mode, validate: :number, default: -1 57 | 58 | # TODO: fix the interval type... 59 | config :stale_cleanup_interval, validate: :number, default: 10 60 | config :stale_cleanup_type, validate: %w[events interval], default: 'events' 61 | 62 | # Should the plugin recover from failure? 63 | # 64 | # If `true`, the plugin will look for temp files from past runs within the 65 | # path (before any dynamic pattern is added) and try to process them 66 | # 67 | # If `false`, the plugin will disregard temp files found 68 | config :recovery, validate: :boolean, default: true 69 | 70 | 71 | # The Kusto endpoint for ingestion related communication. You can see it on the Azure Portal. 72 | config :ingest_url, validate: :string, required: true 73 | 74 | # The following are the credentails used to connect to the Kusto service 75 | # application id 76 | config :app_id, validate: :string, required: false 77 | # application key (secret) 78 | config :app_key, validate: :password, required: false 79 | # aad tenant id 80 | config :app_tenant, validate: :string, default: nil 81 | # managed identity id 82 | config :managed_identity, validate: :string, default: nil 83 | # CLI credentials for dev-test 84 | config :cli_auth, validate: :boolean, default: false 85 | # The following are the data settings that impact where events are written to 86 | # Database name 87 | config :database, validate: :string, required: true 88 | # Target table name 89 | config :table, validate: :string, required: true 90 | # Mapping name - Used by Kusto to map each attribute from incoming event JSON strings to the appropriate column in the table. 91 | # Note that this must be in JSON format, as this is the interface between Logstash and Kusto 92 | # Make this optional as name resolution in the JSON mapping can be done based on attribute names in the incoming event JSON strings 93 | config :json_mapping, validate: :string, default: nil 94 | 95 | # Mapping name - deprecated, use json_mapping 96 | config :mapping, validate: :string, deprecated: true 97 | 98 | 99 | # Determines if local files used for temporary storage will be deleted 100 | # after upload is successful 101 | config :delete_temp_files, validate: :boolean, default: true 102 | 103 | # TODO: will be used to route events to many tables according to event properties 104 | config :dynamic_event_routing, validate: :boolean, default: false 105 | 106 | # Specify how many files can be uploaded concurrently 107 | config :upload_concurrent_count, validate: :number, default: 3 108 | 109 | # Specify how many files can be kept in the upload queue before the main process 110 | # starts processing them in the main thread (not healthy) 111 | config :upload_queue_size, validate: :number, default: 30 112 | 113 | # Host of the proxy , is an optional field. Can connect directly 114 | config :proxy_host, validate: :string, required: false 115 | 116 | # Port where the proxy runs , defaults to 80. Usually a value like 3128 117 | config :proxy_port, validate: :number, required: false , default: 80 118 | 119 | # Check Proxy URL can be over http or https. Dowe need it this way or ignore this & remove this 120 | config :proxy_protocol, validate: :string, required: false , default: 'http' 121 | 122 | default :codec, 'json_lines' 123 | 124 | def register 125 | require 'fileutils' # For mkdir_p 126 | 127 | @files = {} 128 | @io_mutex = Mutex.new 129 | 130 | final_mapping = json_mapping 131 | if final_mapping.nil? || final_mapping.empty? 132 | final_mapping = mapping 133 | end 134 | 135 | # TODO: add id to the tmp path to support multiple outputs of the same type. 136 | # TODO: Fix final_mapping when dynamic routing is supported 137 | # add fields from the meta that will note the destination of the events in the file 138 | @path = if dynamic_event_routing 139 | File.expand_path("#{path}.%{[@metadata][database]}.%{[@metadata][table]}.%{[@metadata][final_mapping]}") 140 | else 141 | File.expand_path("#{path}.#{database}.#{table}") 142 | end 143 | 144 | validate_path 145 | 146 | @file_root = if path_with_field_ref? 147 | extract_file_root 148 | else 149 | File.dirname(path) 150 | end 151 | @failure_path = File.join(@file_root, @filename_failure) 152 | 153 | executor = Concurrent::ThreadPoolExecutor.new(min_threads: 1, 154 | max_threads: upload_concurrent_count, 155 | max_queue: upload_queue_size, 156 | fallback_policy: :caller_runs) 157 | 158 | @ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, managed_identity, cli_auth, database, table, final_mapping, delete_temp_files, proxy_host, proxy_port,proxy_protocol, @logger, executor) 159 | 160 | # send existing files 161 | recover_past_files if recovery 162 | 163 | @last_stale_cleanup_cycle = Time.now 164 | 165 | @flush_interval = @flush_interval.to_i 166 | if @flush_interval > 0 167 | @flusher = Interval.start(@flush_interval, -> { flush_pending_files }) 168 | end 169 | 170 | if (@stale_cleanup_type == 'interval') && (@stale_cleanup_interval > 0) 171 | @cleaner = Interval.start(stale_cleanup_interval, -> { close_stale_files }) 172 | end 173 | end 174 | 175 | private 176 | def validate_path 177 | if (root_directory =~ FIELD_REF) != nil 178 | @logger.error('The starting part of the path should not be dynamic.', path: @path) 179 | raise LogStash::ConfigurationError.new('The starting part of the path should not be dynamic.') 180 | end 181 | 182 | if !path_with_field_ref? 183 | @logger.error('Path should include some time related fields to allow for file rotation.', path: @path) 184 | raise LogStash::ConfigurationError.new('Path should include some time related fields to allow for file rotation.') 185 | end 186 | end 187 | 188 | private 189 | def root_directory 190 | parts = @path.split(File::SEPARATOR).reject(&:empty?) 191 | if Gem.win_platform? 192 | # First part is the drive letter 193 | parts[1] 194 | else 195 | parts.first 196 | end 197 | end 198 | 199 | public 200 | def multi_receive_encoded(events_and_encoded) 201 | encoded_by_path = Hash.new { |h, k| h[k] = [] } 202 | 203 | events_and_encoded.each do |event, encoded| 204 | file_output_path = event_path(event) 205 | encoded_by_path[file_output_path] << encoded 206 | end 207 | 208 | @io_mutex.synchronize do 209 | encoded_by_path.each do |path, chunks| 210 | fd = open(path) 211 | # append to the file 212 | chunks.each { |chunk| fd.write(chunk) } 213 | fd.flush unless @flusher && @flusher.alive? 214 | end 215 | 216 | close_stale_files if @stale_cleanup_type == 'events' 217 | end 218 | end 219 | 220 | def close 221 | @flusher.stop unless @flusher.nil? 222 | @cleaner.stop unless @cleaner.nil? 223 | @io_mutex.synchronize do 224 | @logger.debug('Close: closing files') 225 | 226 | @files.each do |path, fd| 227 | begin 228 | fd.close 229 | @logger.debug("Closed file #{path}", fd: fd) 230 | 231 | kusto_send_file(path) 232 | rescue Exception => e 233 | @logger.error('Exception while flushing and closing files.', exception: e) 234 | end 235 | end 236 | end 237 | 238 | @ingestor.stop unless @ingestor.nil? 239 | end 240 | 241 | private 242 | def inside_file_root?(log_path) 243 | target_file = File.expand_path(log_path) 244 | return target_file.start_with?("#{@file_root}/") 245 | end 246 | 247 | private 248 | def event_path(event) 249 | file_output_path = generate_filepath(event) 250 | if path_with_field_ref? && !inside_file_root?(file_output_path) 251 | @logger.warn('The event tried to write outside the files root, writing the event to the failure file', event: event, filename: @failure_path) 252 | file_output_path = @failure_path 253 | elsif !@create_if_deleted && deleted?(file_output_path) 254 | file_output_path = @failure_path 255 | end 256 | @logger.debug('Writing event to tmp file.', filename: file_output_path) 257 | 258 | file_output_path 259 | end 260 | 261 | private 262 | def generate_filepath(event) 263 | event.sprintf(@path) 264 | end 265 | 266 | private 267 | def path_with_field_ref? 268 | path =~ FIELD_REF 269 | end 270 | 271 | private 272 | def extract_file_root 273 | parts = File.expand_path(path).split(File::SEPARATOR) 274 | parts.take_while { |part| part !~ FIELD_REF }.join(File::SEPARATOR) 275 | end 276 | 277 | # the back-bone of @flusher, our periodic-flushing interval. 278 | private 279 | def flush_pending_files 280 | @io_mutex.synchronize do 281 | @logger.debug('Starting flush cycle') 282 | 283 | @files.each do |path, fd| 284 | @logger.debug('Flushing file', path: path, fd: fd) 285 | fd.flush 286 | end 287 | end 288 | rescue Exception => e 289 | # squash exceptions caught while flushing after logging them 290 | @logger.error('Exception flushing files', exception: e.message, backtrace: e.backtrace) 291 | end 292 | 293 | # every 10 seconds or so (triggered by events, but if there are no events there's no point closing files anyway) 294 | private 295 | def close_stale_files 296 | now = Time.now 297 | return unless now - @last_stale_cleanup_cycle >= @stale_cleanup_interval 298 | 299 | @logger.debug('Starting stale files cleanup cycle', files: @files) 300 | inactive_files = @files.select { |path, fd| not fd.active } 301 | @logger.debug("#{inactive_files.count} stale files found", inactive_files: inactive_files) 302 | inactive_files.each do |path, fd| 303 | @logger.info("Closing file #{path}") 304 | fd.close 305 | @files.delete(path) 306 | 307 | kusto_send_file(path) 308 | end 309 | # mark all files as inactive, a call to write will mark them as active again 310 | @files.each { |path, fd| fd.active = false } 311 | @last_stale_cleanup_cycle = now 312 | end 313 | 314 | private 315 | def cached?(path) 316 | @files.include?(path) && !@files[path].nil? 317 | end 318 | 319 | private 320 | def deleted?(path) 321 | !File.exist?(path) 322 | end 323 | 324 | private 325 | def open(path) 326 | return @files[path] if !deleted?(path) && cached?(path) 327 | 328 | if deleted?(path) 329 | if @create_if_deleted 330 | @logger.debug('Required file does not exist, creating it.', path: path) 331 | @files.delete(path) 332 | else 333 | return @files[path] if cached?(path) 334 | end 335 | end 336 | 337 | @logger.info('Opening file', path: path) 338 | 339 | dir = File.dirname(path) 340 | if !Dir.exist?(dir) 341 | @logger.info('Creating directory', directory: dir) 342 | if @dir_mode != -1 343 | FileUtils.mkdir_p(dir, mode: @dir_mode) 344 | else 345 | FileUtils.mkdir_p(dir) 346 | end 347 | end 348 | 349 | # work around a bug opening fifos (bug JRUBY-6280) 350 | stat = begin 351 | File.stat(path) 352 | rescue 353 | nil 354 | end 355 | fd = if stat && stat.ftype == 'fifo' && LogStash::Environment.jruby? 356 | java.io.FileWriter.new(java.io.File.new(path)) 357 | elsif @file_mode != -1 358 | File.new(path, 'a+', @file_mode) 359 | else 360 | File.new(path, 'a+') 361 | end 362 | # fd = if @file_mode != -1 363 | # File.new(path, 'a+', @file_mode) 364 | # else 365 | # File.new(path, 'a+') 366 | # end 367 | # end 368 | @files[path] = IOWriter.new(fd) 369 | end 370 | 371 | private 372 | def kusto_send_file(file_path) 373 | @ingestor.upload_async(file_path, delete_temp_files) 374 | end 375 | 376 | private 377 | def recover_past_files 378 | require 'find' 379 | 380 | # we need to find the last "regular" part in the path before any dynamic vars 381 | path_last_char = @path.length - 1 382 | 383 | pattern_start = @path.index('%') || path_last_char 384 | last_folder_before_pattern = @path.rindex('/', pattern_start) || path_last_char 385 | new_path = path[0..last_folder_before_pattern] 386 | 387 | begin 388 | return unless Dir.exist?(new_path) 389 | @logger.info("Going to recover old files in path #{@new_path}") 390 | 391 | old_files = Find.find(new_path).select { |p| /.*\.#{database}\.#{table}$/ =~ p } 392 | @logger.info("Found #{old_files.length} old file(s), sending them now...") 393 | 394 | old_files.each do |file| 395 | kusto_send_file(file) 396 | end 397 | rescue Errno::ENOENT => e 398 | @logger.warn('No such file or directory', exception: e.class, message: e.message, path: new_path, backtrace: e.backtrace) 399 | end 400 | end 401 | end 402 | 403 | # wrapper class 404 | class IOWriter 405 | def initialize(io) 406 | @io = io 407 | end 408 | 409 | def write(*args) 410 | @io.write(*args) 411 | @active = true 412 | end 413 | 414 | def flush 415 | @io.flush 416 | end 417 | 418 | def method_missing(method_name, *args, &block) 419 | if @io.respond_to?(method_name) 420 | 421 | @io.send(method_name, *args, &block) 422 | else 423 | super 424 | end 425 | end 426 | attr_accessor :active 427 | end 428 | -------------------------------------------------------------------------------- /lib/logstash/outputs/kusto/ingestor.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'logstash/outputs/base' 4 | require 'logstash/namespace' 5 | require 'logstash/errors' 6 | 7 | class LogStash::Outputs::Kusto < LogStash::Outputs::Base 8 | ## 9 | # This handles the overall logic and communication with Kusto 10 | # 11 | class Ingestor 12 | require 'logstash-output-kusto_jars' 13 | RETRY_DELAY_SECONDS = 3 14 | DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new( 15 | min_threads: 1, 16 | max_threads: 8, 17 | max_queue: 1, 18 | fallback_policy: :caller_runs 19 | ) 20 | LOW_QUEUE_LENGTH = 3 21 | FIELD_REF = /%\{[^}]+\}/ 22 | 23 | def initialize(ingest_url, app_id, app_key, app_tenant, managed_identity_id, cli_auth, database, table, json_mapping, delete_local, proxy_host , proxy_port , proxy_protocol,logger, threadpool = DEFAULT_THREADPOOL) 24 | @workers_pool = threadpool 25 | @logger = logger 26 | validate_config(database, table, json_mapping,proxy_protocol,app_id, app_key, managed_identity_id,cli_auth) 27 | @logger.info('Preparing Kusto resources.') 28 | 29 | kusto_java = Java::com.microsoft.azure.kusto 30 | apache_http = Java::org.apache.http 31 | # kusto_connection_string = kusto_java.data.auth.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant) 32 | # If there is managed identity, use it. This means the AppId and AppKey are empty/nil 33 | # If there is CLI Auth, use that instead of managed identity 34 | is_managed_identity = (app_id.nil? && app_key.nil? && !cli_auth) 35 | # If it is system managed identity, propagate the system identity 36 | is_system_assigned_managed_identity = is_managed_identity && 0 == "system".casecmp(managed_identity_id) 37 | # Is it direct connection 38 | is_direct_conn = (proxy_host.nil? || proxy_host.empty?) 39 | # Create a connection string 40 | kusto_connection_string = if is_managed_identity 41 | if is_system_assigned_managed_identity 42 | @logger.info('Using system managed identity.') 43 | kusto_java.data.auth.ConnectionStringBuilder.createWithAadManagedIdentity(ingest_url) 44 | else 45 | @logger.info('Using user managed identity.') 46 | kusto_java.data.auth.ConnectionStringBuilder.createWithAadManagedIdentity(ingest_url, managed_identity_id) 47 | end 48 | else 49 | if cli_auth 50 | @logger.warn('*Use of CLI Auth is only for dev-test scenarios. This is ***NOT RECOMMENDED*** for production*') 51 | kusto_java.data.auth.ConnectionStringBuilder.createWithAzureCli(ingest_url) 52 | else 53 | @logger.info('Using app id and app key.') 54 | kusto_java.data.auth.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant) 55 | end 56 | end 57 | # 58 | @logger.debug(Gem.loaded_specs.to_s) 59 | # Unfortunately there's no way to avoid using the gem/plugin name directly... 60 | name_for_tracing = "logstash-output-kusto:#{Gem.loaded_specs['logstash-output-kusto']&.version || "unknown"}" 61 | @logger.debug("Client name for tracing: #{name_for_tracing}") 62 | 63 | tuple_utils = Java::org.apache.commons.lang3.tuple 64 | # kusto_connection_string.setClientVersionForTracing(name_for_tracing) 65 | version_for_tracing=Gem.loaded_specs['logstash-output-kusto']&.version || "unknown" 66 | kusto_connection_string.setConnectorDetails("Logstash",version_for_tracing.to_s,"","",false,"", tuple_utils.Pair.emptyArray()); 67 | 68 | @kusto_client = begin 69 | if is_direct_conn 70 | kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string) 71 | else 72 | kusto_http_client_properties = kusto_java.data.HttpClientProperties.builder().proxy(apache_http.HttpHost.new(proxy_host,proxy_port,proxy_protocol)).build() 73 | kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string, kusto_http_client_properties) 74 | end 75 | end 76 | 77 | @ingestion_properties = kusto_java.ingest.IngestionProperties.new(database, table) 78 | is_mapping_ref_provided = !(json_mapping.nil? || json_mapping.empty?) 79 | if is_mapping_ref_provided 80 | @logger.debug('Using mapping reference.', json_mapping) 81 | @ingestion_properties.setIngestionMapping(json_mapping, kusto_java.ingest.IngestionMapping::IngestionMappingKind::JSON) 82 | @ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DataFormat::JSON) 83 | else 84 | @logger.debug('No mapping reference provided. Columns will be mapped by names in the logstash output') 85 | @ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DataFormat::JSON) 86 | end 87 | @delete_local = delete_local 88 | @logger.debug('Kusto resources are ready.') 89 | end 90 | 91 | def validate_config(database, table, json_mapping, proxy_protocol, app_id, app_key, managed_identity_id,cli_auth) 92 | # Add an additional validation and fail this upfront 93 | if app_id.nil? && app_key.nil? && managed_identity_id.nil? 94 | if cli_auth 95 | @logger.info('Using CLI Auth, this is only for dev-test scenarios. This is ***NOT RECOMMENDED*** for production') 96 | else 97 | @logger.error('managed_identity_id is not provided and app_id/app_key is empty.') 98 | raise LogStash::ConfigurationError.new('managed_identity_id is not provided and app_id/app_key is empty.') 99 | end 100 | end 101 | if database =~ FIELD_REF 102 | @logger.error('database config value should not be dynamic.', database) 103 | raise LogStash::ConfigurationError.new('database config value should not be dynamic.') 104 | end 105 | 106 | if table =~ FIELD_REF 107 | @logger.error('table config value should not be dynamic.', table) 108 | raise LogStash::ConfigurationError.new('table config value should not be dynamic.') 109 | end 110 | 111 | if json_mapping =~ FIELD_REF 112 | @logger.error('json_mapping config value should not be dynamic.', json_mapping) 113 | raise LogStash::ConfigurationError.new('json_mapping config value should not be dynamic.') 114 | end 115 | 116 | if not(["https", "http"].include? proxy_protocol) 117 | @logger.error('proxy_protocol has to be http or https.', proxy_protocol) 118 | raise LogStash::ConfigurationError.new('proxy_protocol has to be http or https.') 119 | end 120 | 121 | end 122 | 123 | def upload_async(path, delete_on_success) 124 | if @workers_pool.remaining_capacity <= LOW_QUEUE_LENGTH 125 | @logger.warn("Ingestor queue capacity is running low with #{@workers_pool.remaining_capacity} free slots.") 126 | end 127 | 128 | @workers_pool.post do 129 | LogStash::Util.set_thread_name("Kusto to ingest file: #{path}") 130 | upload(path, delete_on_success) 131 | end 132 | rescue Exception => e 133 | @logger.error('StandardError.', exception: e.class, message: e.message, path: path, backtrace: e.backtrace) 134 | raise e 135 | end 136 | 137 | def upload(path, delete_on_success) 138 | file_size = File.size(path) 139 | @logger.debug("Sending file to kusto: #{path}. size: #{file_size}") 140 | 141 | # TODO: dynamic routing 142 | # file_metadata = path.partition('.kusto.').last 143 | # file_metadata_parts = file_metadata.split('.') 144 | 145 | # if file_metadata_parts.length == 3 146 | # # this is the number we expect - database, table, json_mapping 147 | # database = file_metadata_parts[0] 148 | # table = file_metadata_parts[1] 149 | # json_mapping = file_metadata_parts[2] 150 | 151 | # local_ingestion_properties = Java::KustoIngestionProperties.new(database, table) 152 | # local_ingestion_properties.addJsonMappingName(json_mapping) 153 | # end 154 | 155 | if file_size > 0 156 | file_source_info = Java::com.microsoft.azure.kusto.ingest.source.FileSourceInfo.new(path, 0); # 0 - let the sdk figure out the size of the file 157 | @kusto_client.ingestFromFile(file_source_info, @ingestion_properties) 158 | else 159 | @logger.warn("File #{path} is an empty file and is not ingested.") 160 | end 161 | File.delete(path) if delete_on_success 162 | @logger.debug("File #{path} sent to kusto.") 163 | rescue Errno::ENOENT => e 164 | @logger.error("File doesn't exist! Unrecoverable error.", exception: e.class, message: e.message, path: path, backtrace: e.backtrace) 165 | rescue Java::JavaNioFile::NoSuchFileException => e 166 | @logger.error("File doesn't exist! Unrecoverable error.", exception: e.class, message: e.message, path: path, backtrace: e.backtrace) 167 | rescue => e 168 | # When the retry limit is reached or another error happen we will wait and retry. 169 | # 170 | # Thread might be stuck here, but I think its better than losing anything 171 | # its either a transient errors or something bad really happened. 172 | @logger.error('Uploading failed, retrying.', exception: e.class, message: e.message, path: path, backtrace: e.backtrace) 173 | sleep RETRY_DELAY_SECONDS 174 | retry 175 | end 176 | 177 | def stop 178 | @workers_pool.shutdown 179 | @workers_pool.wait_for_termination(nil) # block until its done 180 | end 181 | end 182 | end 183 | -------------------------------------------------------------------------------- /lib/logstash/outputs/kusto/interval.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'logstash/outputs/base' 4 | require 'logstash/namespace' 5 | require 'logstash/errors' 6 | 7 | class LogStash::Outputs::Kusto < LogStash::Outputs::Base 8 | ## 9 | # Bare-bones utility for running a block of code at an interval. 10 | # 11 | class Interval 12 | ## 13 | # Initializes a new Interval with the given arguments and starts it 14 | # before returning it. 15 | # 16 | # @param interval [Integer] (see: Interval#initialize) 17 | # @param procsy [#call] (see: Interval#initialize) 18 | # 19 | # @return [Interval] 20 | # 21 | def self.start(interval, procsy) 22 | new(interval, procsy).tap(&:start) 23 | end 24 | 25 | ## 26 | # @param interval [Integer]: time in seconds to wait between calling the given proc 27 | # @param procsy [#call]: proc or lambda to call periodically; must not raise exceptions. 28 | def initialize(interval, procsy) 29 | @interval = interval 30 | @procsy = procsy 31 | 32 | # Mutex, ConditionVariable, etc. 33 | @mutex = Mutex.new 34 | @sleeper = ConditionVariable.new 35 | end 36 | 37 | ## 38 | # Starts the interval, or returns if it has already been started. 39 | # 40 | # @return [void] 41 | def start 42 | @mutex.synchronize do 43 | return if @thread && @thread.alive? 44 | 45 | @thread = Thread.new { run } 46 | end 47 | end 48 | 49 | ## 50 | # Stop the interval. 51 | # Does not interrupt if execution is in-progress. 52 | def stop 53 | @mutex.synchronize do 54 | @stopped = true 55 | end 56 | 57 | @thread && @thread.join 58 | end 59 | 60 | ## 61 | # @return [Boolean] 62 | def alive? 63 | @thread && @thread.alive? 64 | end 65 | 66 | private 67 | 68 | def run 69 | @mutex.synchronize do 70 | loop do 71 | @sleeper.wait(@mutex, @interval) 72 | break if @stopped 73 | 74 | @procsy.call 75 | end 76 | end 77 | ensure 78 | @sleeper.broadcast 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /logstash-output-kusto.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = 'logstash-output-kusto' #WATCH OUT: we hardcoded usage of this name in one of the classes. 3 | s.version = ::File.read('version').split("\n").first 4 | s.licenses = ['Apache-2.0'] 5 | s.summary = 'Writes events to Azure Data Explorer (Kusto)' 6 | s.description = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)' 7 | s.homepage = 'https://github.com/Azure/logstash-output-kusto' 8 | s.authors = ['Tamir Kamara', 'Asaf Mahlev'] 9 | s.email = 'nugetkusto@microsoft.com' 10 | s.require_paths = ["lib", "vendor/jar-dependencies"] 11 | s.platform = 'java' 12 | 13 | # Files 14 | s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*", "vendor/jar-dependencies/**/*.rb", "version", "docs/**/*"] 15 | 16 | # Tests 17 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 18 | 19 | # Special flag to let us know this is actually a logstash plugin 20 | s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } 21 | 22 | # Gem dependencies 23 | s.add_runtime_dependency 'logstash-core', '>= 8.7.0' 24 | s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" 25 | s.add_runtime_dependency 'logstash-codec-json_lines' 26 | s.add_runtime_dependency 'logstash-codec-line' 27 | 28 | s.add_development_dependency 'logstash-devutils' 29 | s.add_development_dependency 'flores' 30 | s.add_development_dependency 'logstash-input-generator' 31 | #s.add_development_dependency 'jar-dependencies', '~> 0.4.1' 32 | s.add_development_dependency 'rspec_junit_formatter' 33 | 34 | 35 | end -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * 6 | * Detailed information about configuring a multi-project build in Gradle can be found 7 | * in the user manual at https://docs.gradle.org/7.2/userguide/multi_project_builds.html 8 | */ 9 | 10 | rootProject.name = 'logstash-output-kusto' 11 | include('lib') 12 | -------------------------------------------------------------------------------- /spec/outputs/kusto/ingestor_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require_relative "../../spec_helpers.rb" 3 | require 'logstash/outputs/kusto' 4 | require 'logstash/outputs/kusto/ingestor' 5 | 6 | describe LogStash::Outputs::Kusto::Ingestor do 7 | 8 | let(:ingest_url) { "https://ingest-sdkse2etest.eastus.kusto.windows.net/" } 9 | let(:app_id) { "myid" } 10 | let(:app_key) { LogStash::Util::Password.new("mykey") } 11 | let(:app_tenant) { "mytenant" } 12 | let(:managed_identity) { "managed_identity" } 13 | let(:database) { "mydatabase" } 14 | let(:cliauth) { false } 15 | let(:table) { "mytable" } 16 | let(:proxy_host) { "localhost" } 17 | let(:proxy_port) { 80 } 18 | let(:proxy_protocol) { "http" } 19 | let(:json_mapping) { "mymapping" } 20 | let(:delete_local) { false } 21 | let(:logger) { spy('logger') } 22 | 23 | describe '#initialize' do 24 | 25 | it 'does not throw an error when initializing' do 26 | # note that this will cause an internal error since connection is being tried. 27 | # however we still want to test that all the java stuff is working as expected 28 | expect { 29 | ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity, cliauth, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger) 30 | ingestor.stop 31 | }.not_to raise_error 32 | end 33 | 34 | dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}'] 35 | 36 | context 'doesnt allow database to have some dynamic part' do 37 | dynamic_name_array.each do |test_database| 38 | it "with database: #{test_database}" do 39 | expect { 40 | ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity, cliauth, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger) 41 | ingestor.stop 42 | }.to raise_error(LogStash::ConfigurationError) 43 | end 44 | end 45 | end 46 | 47 | context 'doesnt allow table to have some dynamic part' do 48 | dynamic_name_array.each do |test_table| 49 | it "with database: #{test_table}" do 50 | expect { 51 | ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity, cliauth, database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger) 52 | ingestor.stop 53 | }.to raise_error(LogStash::ConfigurationError) 54 | end 55 | end 56 | end 57 | 58 | context 'doesnt allow mapping to have some dynamic part' do 59 | dynamic_name_array.each do |json_mapping| 60 | it "with database: #{json_mapping}" do 61 | expect { 62 | ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity, cliauth, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger) 63 | ingestor.stop 64 | }.to raise_error(LogStash::ConfigurationError) 65 | end 66 | end 67 | end 68 | 69 | context 'proxy protocol has to be http or https' do 70 | it "with proxy protocol: socks" do 71 | expect { 72 | ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity, cliauth, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger) 73 | ingestor.stop 74 | }.to raise_error(LogStash::ConfigurationError) 75 | end 76 | end 77 | 78 | context 'one of appid or managedid has to be provided' do 79 | it "with empty managed identity and appid" do 80 | expect { 81 | ingestor = described_class.new(ingest_url, "", app_key, app_tenant, "", cliauth, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger) 82 | ingestor.stop 83 | }.to raise_error(LogStash::ConfigurationError) 84 | end 85 | end 86 | 87 | end 88 | 89 | # describe 'receiving events' do 90 | 91 | # context 'with non-zero flush interval' do 92 | # let(:temporary_output_file) { Stud::Temporary.pathname } 93 | 94 | # let(:event_count) { 100 } 95 | # let(:flush_interval) { 5 } 96 | 97 | # let(:events) do 98 | # event_count.times.map do |idx| 99 | # LogStash::Event.new('subject' => idx) 100 | # end 101 | # end 102 | 103 | # let(:output) { described_class.new(options.merge( {'path' => temporary_output_file, 'flush_interval' => flush_interval, 'delete_temp_files' => false } )) } 104 | 105 | # before(:each) { output.register } 106 | 107 | # after(:each) do 108 | # output.close 109 | # File.exist?(temporary_output_file) && File.unlink(temporary_output_file) 110 | # File.exist?(temporary_output_file + '.kusto') && File.unlink(temporary_output_file + '.kusto') 111 | # end 112 | 113 | # it 'eventually flushes without receiving additional events' do 114 | # output.multi_receive_encoded(events) 115 | 116 | # # events should not all be flushed just yet... 117 | # expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents| 118 | # contents && contents.lines.count < event_count 119 | # end 120 | 121 | # # wait for the flusher to run... 122 | # sleep(flush_interval + 1) 123 | 124 | # # events should all be flushed 125 | # expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents| 126 | # contents && contents.lines.count == event_count 127 | # end 128 | # end 129 | # end 130 | 131 | # end 132 | end 133 | -------------------------------------------------------------------------------- /spec/outputs/kusto_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require 'logstash/outputs/kusto' 3 | require 'logstash/codecs/plain' 4 | require 'logstash/event' 5 | 6 | describe LogStash::Outputs::Kusto do 7 | 8 | let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}", 9 | "ingest_url" => "https://ingest-sdkse2etest.eastus.kusto.windows.net/", 10 | "app_id" => "myid", 11 | "app_key" => "mykey", 12 | "app_tenant" => "mytenant", 13 | "database" => "mydatabase", 14 | "table" => "mytable", 15 | "json_mapping" => "mymapping", 16 | "proxy_host" => "localhost", 17 | "proxy_port" => 3128, 18 | "proxy_protocol" => "https" 19 | } } 20 | 21 | describe '#register' do 22 | 23 | it 'doesnt allow the path to start with a dynamic string' do 24 | kusto = described_class.new(options.merge( {'path' => '/%{name}'} )) 25 | expect { kusto.register }.to raise_error(LogStash::ConfigurationError) 26 | kusto.close 27 | end 28 | 29 | it 'path must include a dynamic string to allow file rotation' do 30 | kusto = described_class.new(options.merge( {'path' => '/{name}'} )) 31 | expect { kusto.register }.to raise_error(LogStash::ConfigurationError) 32 | kusto.close 33 | end 34 | 35 | 36 | dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}'] 37 | 38 | context 'doesnt allow the root directory to have some dynamic part' do 39 | dynamic_name_array.each do |test_path| 40 | it "with path: #{test_path}" do 41 | kusto = described_class.new(options.merge( {'path' => test_path} )) 42 | expect { kusto.register }.to raise_error(LogStash::ConfigurationError) 43 | kusto.close 44 | end 45 | end 46 | end 47 | 48 | it 'allow to have dynamic part after the file root' do 49 | kusto = described_class.new(options.merge({'path' => '/tmp/%{name}'})) 50 | expect { kusto.register }.not_to raise_error 51 | kusto.close 52 | end 53 | 54 | end 55 | 56 | end 57 | -------------------------------------------------------------------------------- /spec/spec_helpers.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "logstash/devutils/rspec/spec_helper" 3 | require "logstash/logging/logger" 4 | 5 | LogStash::Logging::Logger::configure_logging("debug") 6 | 7 | RSpec.configure do |config| 8 | # register around filter that captures stdout and stderr 9 | config.around(:each) do |example| 10 | $stdout = StringIO.new 11 | $stderr = StringIO.new 12 | 13 | example.run 14 | 15 | example.metadata[:stdout] = $stdout.string 16 | example.metadata[:stderr] = $stderr.string 17 | 18 | $stdout = STDOUT 19 | $stderr = STDERR 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /version: -------------------------------------------------------------------------------- 1 | 2.0.9 --------------------------------------------------------------------------------