├── CONTRIBUTORS ├── Gemfile ├── LICENSE ├── NOTICE.TXT ├── README.md ├── Rakefile ├── lib └── logstash │ └── outputs │ ├── amazon_es.rb │ └── amazon_es │ ├── common.rb │ ├── common_configs.rb │ ├── elasticsearch-template-es2x.json │ ├── elasticsearch-template-es5x.json │ ├── elasticsearch-template-es6x.json │ ├── elasticsearch-template-es7x.json │ ├── http_client.rb │ ├── http_client │ ├── manticore_adapter.rb │ └── pool.rb │ ├── http_client_builder.rb │ └── template_manager.rb ├── logstash-output-amazon_es.gemspec └── spec ├── es_spec_helper.rb └── unit ├── http_client_builder_spec.rb └── outputs ├── elasticsearch ├── http_client │ ├── manticore_adapter_spec.rb │ └── pool_spec.rb ├── http_client_spec.rb └── template_manager_spec.rb ├── elasticsearch_spec.rb └── error_whitelist_spec.rb /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The following is a list of people who have contributed ideas, code, bug 2 | reports, or in general have helped logstash along its way. 3 | 4 | Contributors: 5 | * Frank Xu (qinyaox) 6 | * Qingyu Zhou (zhoqingy) 7 | * Ankit Malpani(malpani) 8 | 9 | Note: If you've sent us patches, bug reports, or otherwise contributed to 10 | Logstash, and you aren't on the list above and want to be, please let us know 11 | and we'll make sure you're here. Contributions from folks like you are what make 12 | open source awesome. 13 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | source 'https://rubygems.org' 6 | 7 | gemspec 8 | 9 | logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" 10 | use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" 11 | 12 | if Dir.exist?(logstash_path) && use_logstash_source 13 | gem 'logstash-core', :path => "#{logstash_path}/logstash-core" 14 | gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" 15 | end 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /NOTICE.TXT: -------------------------------------------------------------------------------- 1 | Logstash-output-amazon_es Plugin 2 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | 4 | 5 | This product includes software from the Apache-2.0 licensed Elasticsearch project. 6 | 7 | It requires the following notice: 8 | 9 | Elasticsearch 10 | Copyright 2012-2015 Elasticsearch 11 | Copyright 2020 Elastic and contributors 12 | 13 | This product includes software developed by The Apache Software 14 | Foundation (http://www.apache.org/). 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Logstash Output Plugin 2 | 3 | 4 | This plugin is now in maintenance mode. We will supply bug fixes and security patches for v7.2.X, older versions are no longer supported. This change is because the OpenSearch Project created a new Logstash output plugin 5 | [logstash-output-opensearch](https://github.com/opensearch-project/logstash-output-opensearch) which ships events from 6 | Logstash to OpenSearch 1.x and Elasticsearch 7.x clusters, and also supports SigV4 signing. Having similar functionality 7 | plugins can be redundant, so we plan to eventually replace this logstash-output-amazon_es plugin with the logstash-output-opensearch 8 | plugin. 9 | 10 | To help you migrate to [logstash-output-opensearch](https://github.com/opensearch-project/logstash-output-opensearch) plugin, please 11 | find below a brief migration guide. 12 | 13 | ## Migrating to logstash-output-opensearch plugin 14 | 15 | 16 | This guide provides instructions for existing users of logstash-output-amazon_es plugin to migrate to 17 | logstash-output-opensearch plugin. 18 | 19 | ### Configuration Changes 20 | * The plugin name will change from `amazon_es` to `opensearch`. 21 | * If using HTTPS this must be explicitly configured because `opensearch` plugin does not default to it like `amazon_es` does: 22 | * The protocol must be included in `hosts` as `https` (or option `ssl` added with value `true`) 23 | * `port` must explicitly specified as `443` 24 | * A new parameter `auth_type` will be added to the Config to support SigV4 signing. 25 | * The `region` parameter will move under `auth_type`. 26 | * Credential parameters `aws_access_key_id` and `aws_secret_access_key` will move under `auth_type`. 27 | * The `type` value for `auth_type` for SigV4 signing will be set to `aws_iam`. 28 | 29 | For the Logstash configuration provided in [Configuration for Amazon Elasticsearch Service Output Plugin 30 | ](#configuration-for-amazon-elasticsearch-service-output-plugin), here's a mapped example configuration for 31 | logstash-output-opensearch plugin: 32 | 33 | ``` 34 | output { 35 | opensearch { 36 | hosts => ["https://hostname:port"] 37 | auth_type => { 38 | type => 'aws_iam' 39 | aws_access_key_id => 'ACCESS_KEY' 40 | aws_secret_access_key => 'SECRET_KEY' 41 | region => 'us-west-2' 42 | } 43 | index => "logstash-logs-%{+YYYY.MM.dd}" 44 | } 45 | } 46 | ``` 47 | 48 | ### Installation of logstash-output-opensearch plugin 49 | This [Installation Guide](https://opensearch.org/docs/latest/clients/logstash/index/) has instructions on installing the 50 | logstash-output-opensearch plugin in two ways: Linux (ARM64/X64) OR Docker (ARM64/X64). 51 | 52 | To install the latest version of logstash-output-opensearch, use the normal Logstash plugin installation command: 53 | ```shell 54 | bin/logstash-plugin install logstash-output-opensearch 55 | ``` 56 | 57 | # Using the logstash-output-amazon_es plugin 58 | 59 | 60 | The remainder of this document is for using or developing the logstash-output-amazon_es plugin. 61 | 62 | 63 | ## Overview 64 | 65 | This is a plugin for [Logstash](https://github.com/elastic/logstash) which outputs 66 | to [Amazon OpenSearch Service](https://aws.amazon.com/opensearch-service/) 67 | (successor to Amazon Elasticsearch Service) using 68 | [SigV4 signing](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html). 69 | 70 | ## License 71 | 72 | This library is licensed under Apache License 2.0. 73 | 74 | ## Compatibility 75 | 76 | The following table shows the versions of logstash and logstash-output-amazon_es plugin was built with. 77 | 78 | | logstash-output-amazon_es | Logstash | 79 | |---------------------------|----------| 80 | | 6.0.0 | < 6.0.0 | 81 | | 6.4.2 | >= 6.0.0 | 82 | | 7.0.1 | >= 7.0.0 | 83 | | 7.1.0 | >= 7.0.0 | 84 | | 8.0.0 | >= 7.0.0 | 85 | 86 | Also, logstash-output-amazon_es plugin versions 6.4.0 and newer are tested to be compatible with Elasticsearch 6.5 and greater. 87 | 88 | | logstash-output-amazon_es | Elasticsearch | 89 | | ------------- |----------| 90 | | 6.4.0+ | 6.5+ | 91 | 92 | 93 | ## Installation 94 | 95 | To install the latest version, use the normal Logstash plugin script. 96 | 97 | ```sh 98 | bin/logstash-plugin install logstash-output-amazon_es 99 | ``` 100 | 101 | If you want to use old version of logstash-output-amazon_es, you can use the `--version` 102 | flag to specify the version. For example: 103 | 104 | ```sh 105 | bin/logstash-plugin install --version 6.4.2 logstash-output-amazon_es 106 | ``` 107 | 108 | Starting in 8.0.0, the aws sdk version is bumped to v3. In order for all other AWS plugins to work together, please remove pre-installed plugins and install logstash-integration-aws plugin as follows. See also https://github.com/logstash-plugins/logstash-mixin-aws/issues/38 109 | ``` 110 | # Remove existing logstash aws plugins and install logstash-integration-aws to keep sdk dependency the same 111 | # https://github.com/logstash-plugins/logstash-mixin-aws/issues/38 112 | /usr/share/logstash/bin/logstash-plugin remove logstash-input-s3 113 | /usr/share/logstash/bin/logstash-plugin remove logstash-input-sqs 114 | /usr/share/logstash/bin/logstash-plugin remove logstash-output-s3 115 | /usr/share/logstash/bin/logstash-plugin remove logstash-output-sns 116 | /usr/share/logstash/bin/logstash-plugin remove logstash-output-sqs 117 | /usr/share/logstash/bin/logstash-plugin remove logstash-output-cloudwatch 118 | 119 | /usr/share/logstash/bin/logstash-plugin install --version 0.1.0.pre logstash-integration-aws 120 | bin/logstash-plugin install --version 8.0.0 logstash-output-amazon_es 121 | ``` 122 | 123 | ## Configuration for Amazon Elasticsearch Service Output Plugin 124 | 125 | To run the Logstash Output Amazon Elasticsearch Service plugin, simply add a configuration following the below documentation. 126 | 127 | An example configuration: 128 | 129 | ``` 130 | output { 131 | amazon_es { 132 | hosts => ["foo.us-east-1.es.amazonaws.com"] 133 | region => "us-east-1" 134 | # aws_access_key_id and aws_secret_access_key are optional if instance profile is configured 135 | aws_access_key_id => 'ACCESS_KEY' 136 | aws_secret_access_key => 'SECRET_KEY' 137 | index => "production-logs-%{+YYYY.MM.dd}" 138 | } 139 | } 140 | ``` 141 | 142 | ### Required Parameters 143 | 144 | - hosts (array of string) - the Amazon Elasticsearch Service domain endpoint (e.g. `["foo.us-east-1.es.amazonaws.com"]`) 145 | - region (string, :default => "us-east-1") - region where the domain is located 146 | 147 | ### Optional Parameters 148 | 149 | - Credential parameters: 150 | 151 | * aws_access_key_id, :validate => :string - optional AWS access key 152 | * aws_secret_access_key, :validate => :string - optional AWS secret key 153 | 154 | The credential resolution logic can be described as follows: 155 | 156 | - User passed `aws_access_key_id` and `aws_secret_access_key` in `amazon_es` configuration 157 | - Environment variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET), or `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` (only recognized by Java SDK) 158 | - Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI 159 | - Instance profile credentials delivered through the Amazon EC2 metadata service 160 | 161 | - template (path) - You can set the path to your own template here, if you so desire. If not set, the included template will be used. 162 | - template_name (string, default => "logstash") - defines how the template is named inside Elasticsearch 163 | - port (string, default 443) - Amazon Elasticsearch Service listens on port 443 for HTTPS (default) and port 80 for HTTP. Tweak this value for a custom proxy. 164 | - protocol (string, default https) - The protocol used to connect to the Amazon Elasticsearch Service 165 | - max_bulk_bytes - The max size for a bulk request in bytes. Default is 20MB. It is recommended not to change this value unless needed. For guidance on changing this value, please consult the table for network limits for your instance type: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html#network-limits 166 | 167 | After 6.4.0, users can't set batch size in this output plugin config. However, users can still set batch size in logstash.yml file. 168 | 169 | ### Advanced Optional Parameters 170 | 171 | Starting logstash-output-amazon_es v7.1.0, we have introduced the following optional parameters to resolve specific use cases: 172 | 173 | - service_name (string, default => "es") - Users can define any service name to which the plugin will send a SigV4 signed request 174 | - skip_healthcheck (boolean, default => false) - Boolean to skip healthcheck API and set the major ES version to 7 175 | - skip_template_installation (boolean, default => false) - Boolean to allow users to skip installing templates in usecases that don't require them 176 | 177 | ## Developing 178 | 179 | ### 1. Prerequisites 180 | To get started, you can install JRuby with the Bundler gem using [RVM](https://rvm.io/rvm/install) 181 | 182 | ```shell 183 | rvm install jruby-9.2.5.0 184 | ``` 185 | 186 | ### 2. Plugin Development and Testing 187 | 188 | #### Code 189 | 190 | 1. Verify JRuby is already installed 191 | 192 | ```sh 193 | jruby -v 194 | ``` 195 | 196 | 197 | 2. Install dependencies: 198 | 199 | ```sh 200 | bundle install 201 | ``` 202 | 203 | #### Test 204 | 205 | 1. Update your dependencies: 206 | 207 | ```sh 208 | bundle install 209 | ``` 210 | 211 | 2. Run unit tests: 212 | 213 | ```sh 214 | bundle exec rspec 215 | ``` 216 | 217 | ### 3. Running your unpublished plugin in Logstash 218 | 219 | #### 3.1 Run in a local Logstash clone 220 | 221 | 1. Edit Logstash `Gemfile` and add the local plugin path, for example: 222 | 223 | ```ruby 224 | gem "logstash-output-amazon_es", :path => "/your/local/logstash-output-amazon_es" 225 | ``` 226 | 227 | 2. Install the plugin: 228 | 229 | ```sh 230 | # Logstash 2.3 and higher 231 | bin/logstash-plugin install --no-verify 232 | 233 | # Prior to Logstash 2.3 234 | bin/plugin install --no-verify 235 | ``` 236 | 237 | 3. Run Logstash with your plugin: 238 | 239 | ```sh 240 | bin/logstash -e 'output {amazon_es {}}' 241 | ``` 242 | 243 | At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply re-run Logstash. 244 | 245 | #### 3.2 Run in an installed Logstash 246 | 247 | Before build your `Gemfile`, please make sure use JRuby. Here is how you can know your local Ruby version: 248 | 249 | ```sh 250 | rvm list 251 | ``` 252 | 253 | Please make sure you current using JRuby. Here is how you can change to JRuby 254 | 255 | ```sh 256 | rvm jruby-9.2.5.0 257 | ``` 258 | 259 | You can use the same **3.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory. You can also build the gem and install it using: 260 | 261 | 1. Build your plugin gem: 262 | 263 | ```sh 264 | gem build logstash-output-amazon_es.gemspec 265 | ``` 266 | 267 | 2. Install the plugin from the Logstash home. Please be sure to check the version number against the actual Gem file. Run: 268 | 269 | ```sh 270 | bin/logstash-plugin install /your/local/logstash-output-amazon_es/logstash-output-amazon_es-7.0.1-java.gem 271 | ``` 272 | 273 | 3. Start Logstash and test the plugin. 274 | 275 | 276 | ## Contributing 277 | 278 | All contributions are welcome: ideas, patches, documentation, bug reports, and complaints. 279 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/devutils/rake" 6 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | # encoding: utf-8 6 | require "logstash/namespace" 7 | require "logstash/environment" 8 | require "logstash/outputs/base" 9 | require "logstash/json" 10 | require "concurrent" 11 | require "stud/buffer" 12 | require "socket" # for Socket.gethostname 13 | require "thread" # for safe queueing 14 | require "uri" # for escaping user input 15 | require "forwardable" 16 | 17 | # .Compatibility Note 18 | # [NOTE] 19 | # ================================================================================ 20 | # Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting] 21 | # called `http.content_type.required`. If this option is set to `true`, and you 22 | # are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output 23 | # plugin to version 6.2.5 or higher. 24 | # 25 | # ================================================================================ 26 | # 27 | # This plugin is the recommended method of storing logs in Elasticsearch. 28 | # If you plan on using the Kibana web interface, you'll want to use this output. 29 | # 30 | # This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0. 31 | # We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower, 32 | # yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having 33 | # to upgrade Logstash in lock-step. 34 | # 35 | # You can learn more about Elasticsearch at 36 | # 37 | # ==== Template management for Elasticsearch 5.x 38 | # Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0. 39 | # Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default 40 | # behavior. 41 | # 42 | # ** Users installing ES 5.x and LS 5.x ** 43 | # This change will not affect you and you will continue to use the ES defaults. 44 | # 45 | # ** Users upgrading from LS 2.x to LS 5.x with ES 5.x ** 46 | # LS will not force upgrade the template, if `logstash` template already exists. This means you will still use 47 | # `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after 48 | # the new template is installed. 49 | # 50 | # ==== Retry Policy 51 | # 52 | # The retry policy has changed significantly in the 2.2.0 release. 53 | # This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience 54 | # either partial or total failures. 55 | # 56 | # The following errors are retried infinitely: 57 | # 58 | # - Network errors (inability to connect) 59 | # - 429 (Too many requests) and 60 | # - 503 (Service unavailable) errors 61 | # 62 | # NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions. 63 | # It is more performant for Elasticsearch to retry these exceptions than this plugin. 64 | # 65 | # ==== Batch Sizes ==== 66 | # This plugin attempts to send batches of events as a single request. However, if 67 | # a request exceeds 20MB we will break it up until multiple batch requests. If a single document exceeds 20MB it will be sent as a single request. 68 | # 69 | # ==== DNS Caching 70 | # 71 | # This plugin uses the JVM to lookup DNS entries and is subject to the value of https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl], 72 | # a global setting for the JVM. 73 | # 74 | # As an example, to set your DNS TTL to 1 second you would set 75 | # the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`. 76 | # 77 | # Keep in mind that a connection with keepalive enabled will 78 | # not reevaluate its DNS value while the keepalive is in effect. 79 | # 80 | # ==== HTTP Compression 81 | # 82 | # This plugin supports request and response compression. Response compression is enabled by default and 83 | # for Elasticsearch versions 5.0 and later, the user doesn't have to set any configs in Elasticsearch for 84 | # it to send back compressed response. For versions before 5.0, `http.compression` must be set to `true` in 85 | # Elasticsearch[https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http] to take advantage of response compression when using this plugin 86 | # 87 | # For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression` 88 | # setting in their Logstash config file. 89 | # 90 | class LogStash::Outputs::AmazonElasticSearch < LogStash::Outputs::Base 91 | declare_threadsafe! 92 | 93 | require "logstash/outputs/amazon_es/http_client" 94 | require "logstash/outputs/amazon_es/http_client_builder" 95 | require "logstash/outputs/amazon_es/common_configs" 96 | require "logstash/outputs/amazon_es/common" 97 | 98 | # Protocol agnostic (i.e. non-http, non-java specific) configs go here 99 | include(LogStash::Outputs::AmazonElasticSearch::CommonConfigs) 100 | 101 | # Protocol agnostic methods 102 | include(LogStash::Outputs::AmazonElasticSearch::Common) 103 | 104 | config_name "amazon_es" 105 | 106 | # The Elasticsearch action to perform. Valid actions are: 107 | # 108 | # - index: indexes a document (an event from Logstash). 109 | # - delete: deletes a document by id (An id is required for this action) 110 | # - create: indexes a document, fails if a document by that id already exists in the index. 111 | # - update: updates a document by id. Update has a special case where you can upsert -- update a 112 | # document if not already present. See the `upsert` option. NOTE: This does not work and is not supported 113 | # in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash! 114 | # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}` 115 | # would use the foo field for the action 116 | # 117 | # For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation] 118 | config :action, :validate => :string, :default => "index" 119 | 120 | # Username to authenticate to a secure Elasticsearch cluster 121 | config :user, :validate => :string 122 | # Password to authenticate to a secure Elasticsearch cluster 123 | config :password, :validate => :password 124 | 125 | # You can set the remote port as part of the host, or explicitly here as well 126 | config :port, :validate => :number, :default => 443 127 | 128 | # Sets the protocol thats used to connect to elasticsearch 129 | config :protocol, :validate => :string, :default => "https" 130 | 131 | #Signing specific details 132 | config :region, :validate => :string, :default => "us-east-1" 133 | 134 | #Service name, default is `es` 135 | config :service_name, :validate => :string, :default => "es" 136 | # Credential resolution logic works as follows: 137 | # 138 | # - User passed aws_access_key_id and aws_secret_access_key in aes configuration 139 | # - Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY 140 | # (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET), 141 | # or AWS_ACCESS_KEY and AWS_SECRET_KEY (only recognized by Java SDK) 142 | # - Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI 143 | # - Instance profile credentials delivered through the Amazon EC2 metadata service 144 | config :aws_access_key_id, :validate => :string 145 | config :aws_secret_access_key, :validate => :string 146 | 147 | # HTTP Path at which the Elasticsearch server lives. Use this if you must run Elasticsearch behind a proxy that remaps 148 | # the root path for the Elasticsearch HTTP API lives. 149 | # Note that if you use paths as components of URLs in the 'hosts' field you may 150 | # not also set this field. That will raise an error at startup 151 | config :path, :validate => :string 152 | 153 | # HTTP Path to perform the _bulk requests to 154 | # this defaults to a concatenation of the path parameter and "_bulk" 155 | config :bulk_path, :validate => :string 156 | 157 | # Pass a set of key value pairs as the URL query string. This query string is added 158 | # to every host listed in the 'hosts' configuration. If the 'hosts' list contains 159 | # urls that already have query strings, the one specified here will be appended. 160 | config :parameters, :validate => :hash 161 | 162 | # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme 163 | # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used. 164 | # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts' 165 | config :ssl, :validate => :boolean 166 | 167 | # Option to validate the server's certificate. Disabling this severely compromises security. 168 | # For more information on disabling certificate verification please read 169 | # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf 170 | config :ssl_certificate_verification, :validate => :boolean, :default => true 171 | 172 | # The .cer or .pem file to validate the server's certificate 173 | config :cacert, :validate => :path 174 | 175 | # The JKS truststore to validate the server's certificate. 176 | # Use either `:truststore` or `:cacert` 177 | config :truststore, :validate => :path 178 | 179 | # Set the truststore password 180 | config :truststore_password, :validate => :password 181 | 182 | # The keystore used to present a certificate to the server. 183 | # It can be either .jks or .p12 184 | config :keystore, :validate => :path 185 | 186 | # Set the keystore password 187 | config :keystore_password, :validate => :password 188 | 189 | # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. 190 | # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use 191 | # this with master nodes, you probably want to disable HTTP on them by setting 192 | # `http.enabled` to false in their amazon_es.yml. You can either use the `sniffing` option or 193 | # manually enter multiple Elasticsearch hosts using the `hosts` parameter. 194 | config :sniffing, :validate => :boolean, :default => false 195 | 196 | # How long to wait, in seconds, between sniffing attempts 197 | config :sniffing_delay, :validate => :number, :default => 5 198 | 199 | # HTTP Path to be used for the sniffing requests 200 | # the default value is computed by concatenating the path value and "_nodes/http" 201 | # if sniffing_path is set it will be used as an absolute path 202 | # do not use full URL here, only paths, e.g. "/sniff/_nodes/http" 203 | config :sniffing_path, :validate => :string 204 | 205 | # Set the address of a forward HTTP proxy. 206 | # This used to accept hashes as arguments but now only accepts 207 | # arguments of the URI type to prevent leaking credentials. 208 | config :proxy, :validate => :uri 209 | 210 | # Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If 211 | # a timeout occurs, the request will be retried. 212 | config :timeout, :validate => :number, :default => 60 213 | 214 | # Set the Elasticsearch errors in the whitelist that you don't want to log. 215 | # A useful example is when you want to skip all 409 errors 216 | # which are `document_already_exists_exception`. 217 | config :failure_type_logging_whitelist, :validate => :array, :default => [] 218 | 219 | # While the output tries to reuse connections efficiently we have a maximum. 220 | # This sets the maximum number of open connections the output will create. 221 | # Setting this too low may mean frequently closing / opening connections 222 | # which is bad. 223 | config :pool_max, :validate => :number, :default => 1000 224 | 225 | # While the output tries to reuse connections efficiently we have a maximum per endpoint. 226 | # This sets the maximum number of open connections per endpoint the output will create. 227 | # Setting this too low may mean frequently closing / opening connections 228 | # which is bad. 229 | config :pool_max_per_route, :validate => :number, :default => 100 230 | 231 | # HTTP Path where a HEAD request is sent when a backend is marked down 232 | # the request is sent in the background to see if it has come back again 233 | # before it is once again eligible to service requests. 234 | # If you have custom firewall rules you may need to change this 235 | config :healthcheck_path, :validate => :string 236 | 237 | # How frequently, in seconds, to wait between resurrection attempts. 238 | # Resurrection is the process by which backend endpoints marked 'down' are checked 239 | # to see if they have come back to life 240 | config :resurrect_delay, :validate => :number, :default => 5 241 | 242 | # How long to wait before checking if the connection is stale before executing a request on a connection using keepalive. 243 | # You may want to set this lower, if you get connection errors regularly 244 | # Quoting the Apache commons docs (this client is based Apache Commmons): 245 | # 'Defines period of inactivity in milliseconds after which persistent connections must 246 | # be re-validated prior to being leased to the consumer. Non-positive value passed to 247 | # this method disables connection validation. This check helps detect connections that 248 | # have become stale (half-closed) while kept inactive in the pool.' 249 | # See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info] 250 | config :validate_after_inactivity, :validate => :number, :default => 10000 251 | 252 | # Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond 253 | config :http_compression, :validate => :boolean, :default => false 254 | 255 | # Custom Headers to send on each request to amazon_es nodes 256 | config :custom_headers, :validate => :hash, :default => {} 257 | 258 | #Max bulk size in bytes 259 | config :max_bulk_bytes, :validate => :number, :default => 20 * 1024 * 1024 260 | 261 | #Option for user to skip Healthcheck API for a host when set to True 262 | config :skip_healthcheck, :validate => :boolean, :default => false 263 | 264 | #Allow user to skip installing template when set to True 265 | config :skip_template_installation, :validate => :boolean, :default => false 266 | 267 | def build_client 268 | params["metric"] = metric 269 | @client ||= ::LogStash::Outputs::AmazonElasticSearch::HttpClientBuilder.build(@logger, @hosts, params) 270 | end 271 | 272 | def close 273 | @stopping.make_true 274 | stop_template_installer 275 | @client.close if @client 276 | end 277 | 278 | @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-amazon_es-/ } 279 | 280 | @@plugins.each do |plugin| 281 | name = plugin.name.split('-')[-1] 282 | require "logstash/outputs/amazon_es/#{name}" 283 | end 284 | 285 | end # class LogStash::Outputs::Elasticsearch 286 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/common.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/outputs/amazon_es/template_manager" 6 | 7 | module LogStash; module Outputs; class AmazonElasticSearch; 8 | module Common 9 | attr_reader :client, :hosts, :skip_template_installation 10 | 11 | # These codes apply to documents, not at the request level 12 | DOC_DLQ_CODES = [400, 404] 13 | DOC_SUCCESS_CODES = [200, 201] 14 | DOC_CONFLICT_CODE = 409 15 | 16 | # When you use external versioning, you are communicating that you want 17 | # to ignore conflicts. More obviously, since an external version is a 18 | # constant part of the incoming document, we should not retry, as retrying 19 | # will never succeed. 20 | VERSION_TYPES_PERMITTING_CONFLICT = ["external", "external_gt", "external_gte"] 21 | 22 | def register 23 | @template_installed = Concurrent::AtomicBoolean.new(false) 24 | @stopping = Concurrent::AtomicBoolean.new(false) 25 | # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior. 26 | @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil 27 | 28 | setup_hosts # properly sets @hosts 29 | build_client 30 | check_action_validity 31 | @bulk_request_metrics = metric.namespace(:bulk_requests) 32 | @document_level_metrics = metric.namespace(:documents) 33 | install_template_after_successful_connection 34 | @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s)) 35 | end 36 | 37 | # Receive an array of events and immediately attempt to index them (no buffering) 38 | def multi_receive(events) 39 | until @template_installed.true? 40 | sleep 1 41 | end 42 | retrying_submit(events.map {|e| event_action_tuple(e)}) 43 | end 44 | 45 | def install_template_after_successful_connection 46 | @template_installer ||= Thread.new do 47 | sleep_interval = @retry_initial_interval 48 | until successful_connection? || @stopping.true? 49 | @logger.debug("Waiting for connectivity to Elasticsearch cluster. Retrying in #{sleep_interval}s") 50 | Stud.stoppable_sleep(sleep_interval) { @stopping.true? } 51 | sleep_interval = next_sleep_interval(sleep_interval) 52 | end 53 | install_template if successful_connection? 54 | end 55 | end 56 | 57 | def stop_template_installer 58 | @template_installer.join unless @template_installer.nil? 59 | end 60 | 61 | def successful_connection? 62 | !!maximum_seen_major_version 63 | end 64 | 65 | # Convert the event into a 3-tuple of action, params, and event 66 | def event_action_tuple(event) 67 | 68 | action = event.sprintf(@action) 69 | 70 | params = { 71 | :_id => @document_id ? event.sprintf(@document_id) : nil, 72 | :_index => event.sprintf(@index), 73 | :_type => get_event_type(event), 74 | :_routing => @routing ? event.sprintf(@routing) : nil 75 | } 76 | 77 | if @pipeline 78 | params[:pipeline] = event.sprintf(@pipeline) 79 | end 80 | 81 | if @parent 82 | if @join_field 83 | join_value = event.get(@join_field) 84 | parent_value = event.sprintf(@parent) 85 | event.set(@join_field, { "name" => join_value, "parent" => parent_value }) 86 | params[:_routing] = event.sprintf(@parent) 87 | else 88 | params[:parent] = event.sprintf(@parent) 89 | end 90 | end 91 | 92 | if action == 'update' 93 | params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != "" 94 | params[:_script] = event.sprintf(@script) if @script != "" 95 | params[:_retry_on_conflict] = @retry_on_conflict 96 | end 97 | 98 | if @version 99 | params[:version] = event.sprintf(@version) 100 | end 101 | 102 | if @version_type 103 | params[:version_type] = event.sprintf(@version_type) 104 | end 105 | 106 | [action, params, event] 107 | end 108 | 109 | def setup_hosts 110 | @hosts = Array(@hosts) 111 | if @hosts.empty? 112 | @logger.info("No 'host' set in amazon_es output. Defaulting to localhost") 113 | @hosts.replace(["localhost"]) 114 | end 115 | end 116 | 117 | def maximum_seen_major_version 118 | client.maximum_seen_major_version 119 | end 120 | 121 | def install_template 122 | if skip_template_installation == false 123 | TemplateManager.install_template(self) 124 | @template_installed.make_true 125 | elsif skip_template_installation == true 126 | @template_installed.make_true 127 | end 128 | end 129 | 130 | def check_action_validity 131 | raise LogStash::ConfigurationError, "No action specified!" unless @action 132 | 133 | # If we're using string interpolation, we're good! 134 | return if @action =~ /%{.+}/ 135 | return if valid_actions.include?(@action) 136 | 137 | raise LogStash::ConfigurationError, "Action '#{@action}' is invalid! Pick one of #{valid_actions} or use a sprintf style statement" 138 | end 139 | 140 | # To be overidden by the -java version 141 | VALID_HTTP_ACTIONS=["index", "delete", "create", "update"] 142 | def valid_actions 143 | VALID_HTTP_ACTIONS 144 | end 145 | 146 | def retrying_submit(actions) 147 | # Initially we submit the full list of actions 148 | submit_actions = actions 149 | 150 | sleep_interval = @retry_initial_interval 151 | 152 | while submit_actions && submit_actions.length > 0 153 | 154 | # We retry with whatever is didn't succeed 155 | begin 156 | submit_actions = submit(submit_actions) 157 | if submit_actions && submit_actions.size > 0 158 | @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request.", :count => submit_actions.size) 159 | end 160 | rescue => e 161 | @logger.error("Encountered an unexpected error submitting a bulk request! Will retry.", 162 | :error_message => e.message, 163 | :class => e.class.name, 164 | :backtrace => e.backtrace) 165 | end 166 | 167 | # Everything was a success! 168 | break if !submit_actions || submit_actions.empty? 169 | 170 | # If we're retrying the action sleep for the recommended interval 171 | # Double the interval for the next time through to achieve exponential backoff 172 | Stud.stoppable_sleep(sleep_interval) { @stopping.true? } 173 | sleep_interval = next_sleep_interval(sleep_interval) 174 | end 175 | end 176 | 177 | def sleep_for_interval(sleep_interval) 178 | Stud.stoppable_sleep(sleep_interval) { @stopping.true? } 179 | next_sleep_interval(sleep_interval) 180 | end 181 | 182 | def next_sleep_interval(current_interval) 183 | doubled = current_interval * 2 184 | doubled > @retry_max_interval ? @retry_max_interval : doubled 185 | end 186 | 187 | def submit(actions) 188 | bulk_response = safe_bulk(actions) 189 | 190 | # If the response is nil that means we were in a retry loop 191 | # and aborted since we're shutting down 192 | return if bulk_response.nil? 193 | 194 | # If it did return and there are no errors we're good as well 195 | if bulk_response["errors"] 196 | @bulk_request_metrics.increment(:with_errors) 197 | else 198 | @bulk_request_metrics.increment(:successes) 199 | @document_level_metrics.increment(:successes, actions.size) 200 | return 201 | end 202 | 203 | actions_to_retry = [] 204 | bulk_response["items"].each_with_index do |response,idx| 205 | action_type, action_props = response.first 206 | 207 | status = action_props["status"] 208 | failure = action_props["error"] 209 | action = actions[idx] 210 | action_params = action[1] 211 | 212 | # Retry logic: If it is success, we move on. If it is a failure, we have 3 paths: 213 | # - For 409, we log and drop. there is nothing we can do 214 | # - For a mapping error, we send to dead letter queue for a human to intervene at a later point. 215 | # - For everything else there's mastercard. Yep, and we retry indefinitely. This should fix #572 and other transient network issues 216 | if DOC_SUCCESS_CODES.include?(status) 217 | @document_level_metrics.increment(:successes) 218 | next 219 | elsif DOC_CONFLICT_CODE == status 220 | @document_level_metrics.increment(:non_retryable_failures) 221 | @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"]) 222 | next 223 | elsif DOC_DLQ_CODES.include?(status) 224 | handle_dlq_status("Could not index event to Elasticsearch.", action, status, response) 225 | @document_level_metrics.increment(:non_retryable_failures) 226 | next 227 | else 228 | # only log what the user whitelisted 229 | @document_level_metrics.increment(:retryable_failures) 230 | @logger.info "retrying failed action with response code: #{status} (#{failure})" if !failure_type_logging_whitelist.include?(failure["type"]) 231 | actions_to_retry << action 232 | end 233 | end 234 | 235 | actions_to_retry 236 | end 237 | 238 | def handle_dlq_status(message, action, status, response) 239 | # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior) 240 | if @dlq_writer 241 | # TODO: Change this to send a map with { :status => status, :action => action } in the future 242 | @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}") 243 | else 244 | error_type = response.fetch('index', {}).fetch('error', {})['type'] 245 | if 'invalid_index_name_exception' == error_type 246 | level = :error 247 | else 248 | level = :warn 249 | end 250 | @logger.send level, message, status: status, action: action, response: response 251 | end 252 | end 253 | 254 | # Determine the correct value for the 'type' field for the given event 255 | DEFAULT_EVENT_TYPE_ES6="doc".freeze 256 | DEFAULT_EVENT_TYPE_ES7="_doc".freeze 257 | def get_event_type(event) 258 | # Set the 'type' value for the index. 259 | type = if @document_type 260 | event.sprintf(@document_type) 261 | else 262 | if client.maximum_seen_major_version < 6 263 | event.get("type") || DEFAULT_EVENT_TYPE_ES6 264 | elsif client.maximum_seen_major_version == 6 265 | DEFAULT_EVENT_TYPE_ES6 266 | else 267 | DEFAULT_EVENT_TYPE_ES7 268 | end 269 | end 270 | 271 | if !(type.is_a?(String) || type.is_a?(Numeric)) 272 | @logger.warn("Bad event type! Non-string/integer type value set!", :type_class => type.class, :type_value => type.to_s, :event => event) 273 | end 274 | 275 | type.to_s 276 | end 277 | 278 | # Rescue retryable errors during bulk submission 279 | def safe_bulk(actions) 280 | sleep_interval = @retry_initial_interval 281 | begin 282 | es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]} 283 | response = @client.bulk(es_actions) 284 | response 285 | rescue ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::HostUnreachableError => e 286 | # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL) 287 | # and let the user sort it out from there 288 | @logger.error( 289 | "Attempted to send a bulk request to elasticsearch'"+ 290 | " but Elasticsearch appears to be unreachable or down!", 291 | :error_message => e.message, 292 | :class => e.class.name, 293 | :will_retry_in_seconds => sleep_interval 294 | ) 295 | @logger.debug("Failed actions for last bad bulk request!", :actions => actions) 296 | 297 | # We retry until there are no errors! Errors should all go to the retry queue 298 | sleep_interval = sleep_for_interval(sleep_interval) 299 | @bulk_request_metrics.increment(:failures) 300 | retry unless @stopping.true? 301 | rescue ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e 302 | @logger.error( 303 | "Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down?", 304 | :error_message => e.message, 305 | :class => e.class.name, 306 | :will_retry_in_seconds => sleep_interval 307 | ) 308 | Stud.stoppable_sleep(sleep_interval) { @stopping.true? } 309 | sleep_interval = next_sleep_interval(sleep_interval) 310 | @bulk_request_metrics.increment(:failures) 311 | retry unless @stopping.true? 312 | rescue ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError => e 313 | @bulk_request_metrics.increment(:failures) 314 | log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s} 315 | log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose 316 | message = "Encountered a retryable error. Will Retry with exponential backoff " 317 | 318 | # We treat 429s as a special case because these really aren't errors, but 319 | # rather just ES telling us to back off a bit, which we do. 320 | # The other retryable code is 503, which are true errors 321 | # Even though we retry the user should be made aware of these 322 | if e.response_code == 429 323 | logger.debug(message, log_hash) 324 | else 325 | logger.error(message, log_hash) 326 | end 327 | 328 | sleep_interval = sleep_for_interval(sleep_interval) 329 | retry 330 | rescue => e 331 | # Stuff that should never happen 332 | # For all other errors print out full connection issues 333 | @logger.error( 334 | "An unknown error occurred sending a bulk request to Elasticsearch. We will retry indefinitely", 335 | :error_message => e.message, 336 | :error_class => e.class.name, 337 | :backtrace => e.backtrace 338 | ) 339 | 340 | @logger.debug("Failed actions for last bad bulk request!", :actions => actions) 341 | 342 | sleep_interval = sleep_for_interval(sleep_interval) 343 | @bulk_request_metrics.increment(:failures) 344 | retry unless @stopping.true? 345 | end 346 | end 347 | 348 | def dlq_enabled? 349 | # TODO there should be a better way to query if DLQ is enabled 350 | # See more in: https://github.com/elastic/logstash/issues/8064 351 | respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) && 352 | !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter) 353 | end 354 | end 355 | end; end; end 356 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/common_configs.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require 'forwardable' # Needed for logstash core SafeURI. We need to patch this in core: https://github.com/elastic/logstash/pull/5978 6 | 7 | module LogStash; module Outputs; class AmazonElasticSearch 8 | module CommonConfigs 9 | def self.included(mod) 10 | # The index to write events to. This can be dynamic using the `%{foo}` syntax. 11 | # The default value will partition your indices by day so you can more easily 12 | # delete old data or only search specific date ranges. 13 | # Indexes may not contain uppercase characters. 14 | # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. 15 | # LS uses Joda to format the index pattern from event timestamp. 16 | # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here]. 17 | mod.config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}" 18 | 19 | mod.config :document_type, 20 | :validate => :string, 21 | :deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature" 22 | 23 | # From Logstash 1.3 onwards, a template is applied to Elasticsearch during 24 | # Logstash's startup if one with the name `template_name` does not already exist. 25 | # By default, the contents of this template is the default template for 26 | # `logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern 27 | # `logstash-*`. Should you require support for other index names, or would like 28 | # to change the mappings in the template in general, a custom template can be 29 | # specified by setting `template` to the path of a template file. 30 | # 31 | # Setting `manage_template` to false disables this feature. If you require more 32 | # control over template creation, (e.g. creating indices dynamically based on 33 | # field names) you should set `manage_template` to false and use the REST 34 | # API to apply your templates manually. 35 | mod.config :manage_template, :validate => :boolean, :default => true 36 | 37 | # This configuration option defines how the template is named inside Elasticsearch. 38 | # Note that if you have used the template management features and subsequently 39 | # change this, you will need to prune the old template manually, e.g. 40 | # 41 | # `curl -XDELETE ` 42 | # 43 | # where `OldTemplateName` is whatever the former setting was. 44 | mod.config :template_name, :validate => :string, :default => "logstash" 45 | 46 | # You can set the path to your own template here, if you so desire. 47 | # If not set, the included template will be used. 48 | mod.config :template, :validate => :path 49 | 50 | # The template_overwrite option will always overwrite the indicated template 51 | # in Elasticsearch with either the one indicated by template or the included one. 52 | # This option is set to false by default. If you always want to stay up to date 53 | # with the template provided by Logstash, this option could be very useful to you. 54 | # Likewise, if you have your own template file managed by puppet, for example, and 55 | # you wanted to be able to update it regularly, this option could help there as well. 56 | # 57 | # Please note that if you are using your own customized version of the Logstash 58 | # template (logstash), setting this to true will make Logstash to overwrite 59 | # the "logstash" template (i.e. removing all customized settings) 60 | mod.config :template_overwrite, :validate => :boolean, :default => false 61 | 62 | # The document ID for the index. Useful for overwriting existing entries in 63 | # Elasticsearch with the same ID. 64 | mod.config :document_id, :validate => :string 65 | 66 | # The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here. 67 | # See https://www.elastic.co/blog/elasticsearch-versioning-support. 68 | mod.config :version, :validate => :string 69 | 70 | # The version_type to use for indexing. 71 | # See https://www.elastic.co/blog/elasticsearch-versioning-support. 72 | # See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#_version_types 73 | mod.config :version_type, :validate => ["internal", 'external', "external_gt", "external_gte", "force"] 74 | 75 | # A routing override to be applied to all processed events. 76 | # This can be dynamic using the `%{foo}` syntax. 77 | mod.config :routing, :validate => :string 78 | 79 | # For child documents, ID of the associated parent. 80 | # This can be dynamic using the `%{foo}` syntax. 81 | mod.config :parent, :validate => :string, :default => nil 82 | 83 | # For child documents, name of the join field 84 | mod.config :join_field, :validate => :string, :default => nil 85 | 86 | # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter. 87 | # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300). 88 | # `"127.0.0.1"` 89 | # `["127.0.0.1:9200","127.0.0.2:9200"]` 90 | # `["http://127.0.0.1"]` 91 | # `["https://127.0.0.1:9200"]` 92 | # `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath) 93 | # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list 94 | # to prevent LS from sending bulk requests to the master nodes. So this parameter should only reference either data or client nodes in Elasticsearch. 95 | # 96 | # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance. 97 | mod.config :hosts, :validate => :uri, :default => [::LogStash::Util::SafeURI.new("//127.0.0.1")], :list => true 98 | 99 | mod.config :flush_size, :validate => :number, :obsolete => "This setting is no longer available as we now try to restrict bulk requests to sane sizes. See the 'Batch Sizes' section of the docs. If you think you still need to restrict payloads based on the number, not size, of events, please open a ticket." 100 | 101 | mod.config :idle_flush_time, :validate => :number, :obsolete => "This settings is no longer valid. This was a no-op now as every pipeline batch is flushed synchronously obviating the need for this option." 102 | 103 | # Set upsert content for update mode.s 104 | # Create a new document with this parameter as json string if `document_id` doesn't exists 105 | mod.config :upsert, :validate => :string, :default => "" 106 | 107 | # Enable `doc_as_upsert` for update mode. 108 | # Create a new document with source if `document_id` doesn't exist in Elasticsearch 109 | mod.config :doc_as_upsert, :validate => :boolean, :default => false 110 | 111 | # Set script name for scripted update mode 112 | mod.config :script, :validate => :string, :default => "" 113 | 114 | # Define the type of script referenced by "script" variable 115 | # inline : "script" contains inline script 116 | # indexed : "script" contains the name of script directly indexed in amazon_es 117 | # file : "script" contains the name of script stored in elasticseach's config directory 118 | mod.config :script_type, :validate => ["inline", 'indexed', "file"], :default => ["inline"] 119 | 120 | # Set the language of the used script. If not set, this defaults to painless in ES 5.0 121 | mod.config :script_lang, :validate => :string, :default => "painless" 122 | 123 | # Set variable name passed to script (scripted update) 124 | mod.config :script_var_name, :validate => :string, :default => "event" 125 | 126 | # if enabled, script is in charge of creating non-existent document (scripted update) 127 | mod.config :scripted_upsert, :validate => :boolean, :default => false 128 | 129 | # Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval` 130 | mod.config :retry_initial_interval, :validate => :number, :default => 2 131 | 132 | # Set max interval in seconds between bulk retries. 133 | mod.config :retry_max_interval, :validate => :number, :default => 64 134 | 135 | # The number of times Elasticsearch should internally retry an update/upserted document 136 | # See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates] 137 | # for more info 138 | mod.config :retry_on_conflict, :validate => :number, :default => 1 139 | 140 | # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration 141 | # here like `pipeline => "%{INGEST_PIPELINE}"` 142 | mod.config :pipeline, :validate => :string, :default => nil 143 | end 144 | end 145 | end end end 146 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/elasticsearch-template-es2x.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "logstash-*", 3 | "settings" : { 4 | "index.refresh_interval" : "5s" 5 | }, 6 | "mappings" : { 7 | "_default_" : { 8 | "_all" : {"enabled" : true, "omit_norms" : true}, 9 | "dynamic_templates" : [ { 10 | "message_field" : { 11 | "path_match" : "message", 12 | "match_mapping_type" : "string", 13 | "mapping" : { 14 | "type" : "string", "index" : "analyzed", "omit_norms" : true, 15 | "fielddata" : { "format" : "disabled" } 16 | } 17 | } 18 | }, { 19 | "string_fields" : { 20 | "match" : "*", 21 | "match_mapping_type" : "string", 22 | "mapping" : { 23 | "type" : "string", "index" : "analyzed", "omit_norms" : true, 24 | "fielddata" : { "format" : "disabled" }, 25 | "fields" : { 26 | "raw" : {"type": "string", "index" : "not_analyzed", "doc_values" : true, "ignore_above" : 256} 27 | } 28 | } 29 | } 30 | }, { 31 | "float_fields" : { 32 | "match" : "*", 33 | "match_mapping_type" : "float", 34 | "mapping" : { "type" : "float", "doc_values" : true } 35 | } 36 | }, { 37 | "double_fields" : { 38 | "match" : "*", 39 | "match_mapping_type" : "double", 40 | "mapping" : { "type" : "double", "doc_values" : true } 41 | } 42 | }, { 43 | "byte_fields" : { 44 | "match" : "*", 45 | "match_mapping_type" : "byte", 46 | "mapping" : { "type" : "byte", "doc_values" : true } 47 | } 48 | }, { 49 | "short_fields" : { 50 | "match" : "*", 51 | "match_mapping_type" : "short", 52 | "mapping" : { "type" : "short", "doc_values" : true } 53 | } 54 | }, { 55 | "integer_fields" : { 56 | "match" : "*", 57 | "match_mapping_type" : "integer", 58 | "mapping" : { "type" : "integer", "doc_values" : true } 59 | } 60 | }, { 61 | "long_fields" : { 62 | "match" : "*", 63 | "match_mapping_type" : "long", 64 | "mapping" : { "type" : "long", "doc_values" : true } 65 | } 66 | }, { 67 | "date_fields" : { 68 | "match" : "*", 69 | "match_mapping_type" : "date", 70 | "mapping" : { "type" : "date", "doc_values" : true } 71 | } 72 | }, { 73 | "geo_point_fields" : { 74 | "match" : "*", 75 | "match_mapping_type" : "geo_point", 76 | "mapping" : { "type" : "geo_point", "doc_values" : true } 77 | } 78 | } ], 79 | "properties" : { 80 | "@timestamp": { "type": "date", "doc_values" : true }, 81 | "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true }, 82 | "geoip" : { 83 | "type" : "object", 84 | "dynamic": true, 85 | "properties" : { 86 | "ip": { "type": "ip", "doc_values" : true }, 87 | "location" : { "type" : "geo_point", "doc_values" : true }, 88 | "latitude" : { "type" : "float", "doc_values" : true }, 89 | "longitude" : { "type" : "float", "doc_values" : true } 90 | } 91 | } 92 | } 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/elasticsearch-template-es5x.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "logstash-*", 3 | "version" : 50001, 4 | "settings" : { 5 | "index.refresh_interval" : "5s" 6 | }, 7 | "mappings" : { 8 | "_default_" : { 9 | "_all" : {"enabled" : true, "norms" : false}, 10 | "dynamic_templates" : [ { 11 | "message_field" : { 12 | "path_match" : "message", 13 | "match_mapping_type" : "string", 14 | "mapping" : { 15 | "type" : "text", 16 | "norms" : false 17 | } 18 | } 19 | }, { 20 | "string_fields" : { 21 | "match" : "*", 22 | "match_mapping_type" : "string", 23 | "mapping" : { 24 | "type" : "text", "norms" : false, 25 | "fields" : { 26 | "keyword" : { "type": "keyword", "ignore_above": 256 } 27 | } 28 | } 29 | } 30 | } ], 31 | "properties" : { 32 | "@timestamp": { "type": "date", "include_in_all": false }, 33 | "@version": { "type": "keyword", "include_in_all": false }, 34 | "geoip" : { 35 | "dynamic": true, 36 | "properties" : { 37 | "ip": { "type": "ip" }, 38 | "location" : { "type" : "geo_point" }, 39 | "latitude" : { "type" : "half_float" }, 40 | "longitude" : { "type" : "half_float" } 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/elasticsearch-template-es6x.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "logstash-*", 3 | "version" : 60001, 4 | "settings" : { 5 | "index.refresh_interval" : "5s" 6 | }, 7 | "mappings" : { 8 | "_default_" : { 9 | "dynamic_templates" : [ { 10 | "message_field" : { 11 | "path_match" : "message", 12 | "match_mapping_type" : "string", 13 | "mapping" : { 14 | "type" : "text", 15 | "norms" : false 16 | } 17 | } 18 | }, { 19 | "string_fields" : { 20 | "match" : "*", 21 | "match_mapping_type" : "string", 22 | "mapping" : { 23 | "type" : "text", "norms" : false, 24 | "fields" : { 25 | "keyword" : { "type": "keyword", "ignore_above": 256 } 26 | } 27 | } 28 | } 29 | } ], 30 | "properties" : { 31 | "@timestamp": { "type": "date"}, 32 | "@version": { "type": "keyword"}, 33 | "geoip" : { 34 | "dynamic": true, 35 | "properties" : { 36 | "ip": { "type": "ip" }, 37 | "location" : { "type" : "geo_point" }, 38 | "latitude" : { "type" : "half_float" }, 39 | "longitude" : { "type" : "half_float" } 40 | } 41 | } 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/elasticsearch-template-es7x.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "logstash-*", 3 | "version" : 60002, 4 | "settings" : { 5 | "index.refresh_interval" : "5s", 6 | "number_of_shards": 1 7 | }, 8 | "mappings" : { 9 | "dynamic_templates" : [ { 10 | "message_field" : { 11 | "path_match" : "message", 12 | "match_mapping_type" : "string", 13 | "mapping" : { 14 | "type" : "text", 15 | "norms" : false 16 | } 17 | } 18 | }, { 19 | "string_fields" : { 20 | "match" : "*", 21 | "match_mapping_type" : "string", 22 | "mapping" : { 23 | "type" : "text", "norms" : false, 24 | "fields" : { 25 | "keyword" : { "type": "keyword", "ignore_above": 256 } 26 | } 27 | } 28 | } 29 | } ], 30 | "properties" : { 31 | "@timestamp": { "type": "date"}, 32 | "@version": { "type": "keyword"}, 33 | "geoip" : { 34 | "dynamic": true, 35 | "properties" : { 36 | "ip": { "type": "ip" }, 37 | "location" : { "type" : "geo_point" }, 38 | "latitude" : { "type" : "half_float" }, 39 | "longitude" : { "type" : "half_float" } 40 | } 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/http_client.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/outputs/amazon_es" 6 | require "cabin" 7 | require "base64" 8 | require 'logstash/outputs/amazon_es/http_client/pool' 9 | require 'logstash/outputs/amazon_es/http_client/manticore_adapter' 10 | require 'cgi' 11 | require 'zlib' 12 | require 'stringio' 13 | 14 | module LogStash; module Outputs; class AmazonElasticSearch; 15 | class HttpClient 16 | attr_reader :client, :options, :logger, :pool, :action_count, :recv_count, :max_bulk_bytes 17 | 18 | 19 | # This is here in case we use DEFAULT_OPTIONS in the future 20 | # DEFAULT_OPTIONS = { 21 | # :setting => value 22 | # } 23 | 24 | # 25 | # The `options` is a hash where the following symbol keys have meaning: 26 | # 27 | # * `:hosts` - array of String. Set a list of hosts to use for communication. 28 | # * `:port` - number. set the port to use to communicate with Elasticsearch 29 | # * `:user` - String. The user to use for authentication. 30 | # * `:password` - String. The password to use for authentication. 31 | # * `:timeout` - Float. A duration value, in seconds, after which a socket 32 | # operation or request will be aborted if not yet successful 33 | # * `:client_settings` - a hash; see below for keys. 34 | # 35 | # The `client_settings` key is a has that can contain other settings: 36 | # 37 | # * `:ssl` - Boolean. Enable or disable SSL/TLS. 38 | # * `:proxy` - String. Choose a HTTP HTTProxy to use. 39 | # * `:path` - String. The leading path for prefixing Elasticsearch 40 | # * `:headers` - Hash. Pairs of headers and their values 41 | # requests. This is sometimes used if you are proxying Elasticsearch access 42 | # through a special http path, such as using mod_rewrite. 43 | def initialize(options={}) 44 | @logger = options[:logger] 45 | @metric = options[:metric] 46 | @bulk_request_metrics = @metric.namespace(:bulk_requests) 47 | @bulk_response_metrics = @bulk_request_metrics.namespace(:responses) 48 | @max_bulk_bytes = options[:max_bulk_bytes] 49 | 50 | # Again, in case we use DEFAULT_OPTIONS in the future, uncomment this. 51 | # @options = DEFAULT_OPTIONS.merge(options) 52 | @options = options 53 | 54 | @url_template = build_url_template 55 | puts 'url template' 56 | puts @url_template 57 | 58 | @pool = build_pool(@options) 59 | # mutex to prevent requests and sniffing to access the 60 | # connection pool at the same time 61 | @bulk_path = @options[:bulk_path] 62 | end 63 | 64 | def build_url_template 65 | { 66 | :scheme => self.scheme, 67 | :user => self.user, 68 | :password => self.password, 69 | :host => "URLTEMPLATE", 70 | :port => self.port, 71 | :path => self.path 72 | } 73 | end 74 | 75 | def template_install(name, template, force=false) 76 | if template_exists?(name) && !force 77 | @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name) 78 | return 79 | end 80 | template_put(name, template) 81 | end 82 | 83 | def maximum_seen_major_version 84 | @pool.maximum_seen_major_version 85 | end 86 | 87 | def bulk(actions) 88 | @action_count ||= 0 89 | @action_count += actions.size 90 | 91 | return if actions.empty? 92 | 93 | bulk_actions = actions.collect do |action, args, source| 94 | args, source = update_action_builder(args, source) if action == 'update' 95 | 96 | if source && action != 'delete' 97 | next [ { action => args }, source ] 98 | else 99 | next { action => args } 100 | end 101 | end 102 | 103 | body_stream = StringIO.new 104 | if http_compression 105 | body_stream.set_encoding "BINARY" 106 | stream_writer = Zlib::GzipWriter.new(body_stream, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY) 107 | else 108 | stream_writer = body_stream 109 | end 110 | bulk_responses = [] 111 | bulk_actions.each do |action| 112 | as_json = action.is_a?(Array) ? 113 | action.map {|line| LogStash::Json.dump(line)}.join("\n") : 114 | LogStash::Json.dump(action) 115 | as_json << "\n" 116 | if (body_stream.size + as_json.bytesize) > @max_bulk_bytes 117 | bulk_responses << bulk_send(body_stream) unless body_stream.size == 0 118 | end 119 | stream_writer.write(as_json) 120 | end 121 | stream_writer.close if http_compression 122 | bulk_responses << bulk_send(body_stream) if body_stream.size > 0 123 | body_stream.close if !http_compression 124 | join_bulk_responses(bulk_responses) 125 | end 126 | 127 | def join_bulk_responses(bulk_responses) 128 | { 129 | "errors" => bulk_responses.any? {|r| r["errors"] == true}, 130 | "items" => bulk_responses.reduce([]) {|m,r| m.concat(r.fetch("items", []))} 131 | } 132 | end 133 | 134 | def bulk_send(body_stream) 135 | params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {} 136 | # Discard the URL 137 | response = @pool.post(@bulk_path, params, body_stream.string) 138 | if !body_stream.closed? 139 | body_stream.truncate(0) 140 | body_stream.seek(0) 141 | end 142 | 143 | @bulk_response_metrics.increment(response.code.to_s) 144 | 145 | if response.code != 200 146 | url = ::LogStash::Util::SafeURI.new(response.final_url) 147 | raise ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError.new( 148 | response.code, url, body_stream.to_s, response.body 149 | ) 150 | end 151 | 152 | LogStash::Json.load(response.body) 153 | end 154 | 155 | def get(path) 156 | response = @pool.get(path, nil) 157 | LogStash::Json.load(response.body) 158 | end 159 | 160 | def post(path, params = {}, body_string) 161 | response = @pool.post(path, params, body_string) 162 | LogStash::Json.load(response.body) 163 | end 164 | 165 | def close 166 | @pool.close 167 | end 168 | 169 | def calculate_property(uris, property, default, sniff_check) 170 | values = uris.map(&property).uniq 171 | 172 | if sniff_check && values.size > 1 173 | raise LogStash::ConfigurationError, "Cannot have multiple values for #{property} in hosts when sniffing is enabled!" 174 | end 175 | 176 | uri_value = values.first 177 | 178 | default = nil if default.is_a?(String) && default.empty? # Blanks are as good as nil 179 | uri_value = nil if uri_value.is_a?(String) && uri_value.empty? 180 | 181 | 182 | if default && uri_value && (default != uri_value) 183 | raise LogStash::ConfigurationError, "Explicit value for '#{property}' was declared, but it is different in one of the URLs given! Please make sure your URLs are inline with explicit values. The URLs have the property set to '#{uri_value}', but it was also set to '#{default}' explicitly" 184 | end 185 | 186 | uri_value || default 187 | end 188 | 189 | def sniffing 190 | @options[:sniffing] 191 | end 192 | 193 | def user 194 | calculate_property(uris, :user, @options[:user], sniffing) 195 | end 196 | 197 | def password 198 | calculate_property(uris, :password, @options[:password], sniffing) 199 | end 200 | 201 | def path 202 | calculated = calculate_property(uris, :path, client_settings[:path], sniffing) 203 | calculated = "/#{calculated}" if calculated && !calculated.start_with?("/") 204 | calculated 205 | end 206 | 207 | def scheme 208 | explicit_scheme = if ssl_options && ssl_options.has_key?(:enabled) 209 | ssl_options[:enabled] ? 'https' : 'http' 210 | else 211 | nil 212 | end 213 | 214 | calculated_scheme = calculate_property(uris, :scheme, explicit_scheme, sniffing) 215 | 216 | if calculated_scheme && calculated_scheme !~ /https?/ 217 | raise LogStash::ConfigurationError, "Bad scheme '#{calculated_scheme}' found should be one of http/https" 218 | end 219 | 220 | if calculated_scheme && explicit_scheme && calculated_scheme != explicit_scheme 221 | raise LogStash::ConfigurationError, "SSL option was explicitly set to #{ssl_options[:enabled]} but a URL was also declared with a scheme of '#{explicit_scheme}'. Please reconcile this" 222 | end 223 | 224 | calculated_scheme # May be nil if explicit_scheme is nil! 225 | end 226 | 227 | def port 228 | # We don't set the 'default' here because the default is what the user 229 | # indicated, so we use an || outside of calculate_property. This lets people 230 | # Enter things like foo:123, bar and wind up with foo:123, bar:80 231 | #calculate_property(uris, :port, nil, sniffing) || 9200 232 | calculate_property(uris, :port, @options[:port], sniffing) || 9200 233 | end 234 | 235 | def uris 236 | @options[:hosts] 237 | end 238 | 239 | def client_settings 240 | @options[:client_settings] || {} 241 | end 242 | 243 | def ssl_options 244 | client_settings.fetch(:ssl, {}) 245 | end 246 | 247 | def http_compression 248 | client_settings.fetch(:http_compression, false) 249 | end 250 | 251 | def build_adapter(options) 252 | timeout = options[:timeout] || 0 253 | 254 | adapter_options = { 255 | :socket_timeout => timeout, 256 | :request_timeout => timeout, 257 | } 258 | 259 | adapter_options[:proxy] = client_settings[:proxy] if client_settings[:proxy] 260 | 261 | adapter_options[:check_connection_timeout] = client_settings[:check_connection_timeout] if client_settings[:check_connection_timeout] 262 | 263 | # Having this explicitly set to nil is an error 264 | if client_settings[:pool_max] 265 | adapter_options[:pool_max] = client_settings[:pool_max] 266 | end 267 | 268 | # Having this explicitly set to nil is an error 269 | if client_settings[:pool_max_per_route] 270 | adapter_options[:pool_max_per_route] = client_settings[:pool_max_per_route] 271 | end 272 | 273 | adapter_options[:ssl] = ssl_options if self.scheme == 'https' 274 | 275 | adapter_options[:headers] = client_settings[:headers] if client_settings[:headers] 276 | 277 | adapter_options[:region] = options[:region] 278 | 279 | adapter_options[:service_name] = options[:service_name] 280 | 281 | adapter_options[:port] = options[:port] 282 | 283 | adapter_options[:protocol] = options[:protocol] 284 | 285 | adapter_options[:aws_access_key_id] = options[:aws_access_key_id] 286 | 287 | adapter_options[:aws_secret_access_key] = options[:aws_secret_access_key] 288 | 289 | adapter_class = ::LogStash::Outputs::AmazonElasticSearch::HttpClient::ManticoreAdapter 290 | adapter = adapter_class.new(@logger, adapter_options) 291 | end 292 | 293 | def build_pool(options) 294 | adapter = build_adapter(options) 295 | 296 | pool_options = { 297 | :sniffing => sniffing, 298 | :sniffer_delay => options[:sniffer_delay], 299 | :sniffing_path => options[:sniffing_path], 300 | :healthcheck_path => options[:healthcheck_path], 301 | :resurrect_delay => options[:resurrect_delay], 302 | :url_normalizer => self.method(:host_to_url), 303 | :metric => options[:metric], 304 | :skip_healthcheck => options[:skip_healthcheck], 305 | } 306 | pool_options[:scheme] = self.scheme if self.scheme 307 | 308 | pool_class = ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool 309 | full_urls = @options[:hosts].map {|h| host_to_url(h) } 310 | pool = pool_class.new(@logger, adapter, full_urls, pool_options) 311 | pool.start 312 | pool 313 | end 314 | 315 | def host_to_url(h) 316 | 317 | # Never override the calculated scheme 318 | #raw_scheme = @url_template[:scheme] || 'http' 319 | 320 | raw_scheme = @options[:protocol] || 'https' 321 | 322 | raw_user = h.user || @url_template[:user] 323 | raw_password = h.password || @url_template[:password] 324 | postfixed_userinfo = raw_user && raw_password ? "#{raw_user}:#{raw_password}@" : nil 325 | 326 | raw_host = h.host # Always replace this! 327 | raw_port = h.port || @url_template[:port] 328 | 329 | raw_path = !h.path.nil? && !h.path.empty? && h.path != "/" ? h.path : @url_template[:path] 330 | prefixed_raw_path = raw_path && !raw_path.empty? ? raw_path : "/" 331 | 332 | parameters = client_settings[:parameters] 333 | raw_query = if parameters && !parameters.empty? 334 | combined = h.query ? 335 | Hash[URI::decode_www_form(h.query)].merge(parameters) : 336 | parameters 337 | query_str = combined.flat_map {|k,v| 338 | values = Array(v) 339 | values.map {|av| "#{k}=#{av}"} 340 | }.join("&") 341 | query_str 342 | else 343 | h.query 344 | end 345 | prefixed_raw_query = raw_query && !raw_query.empty? ? "?#{raw_query}" : nil 346 | 347 | raw_url = "#{raw_scheme}://#{postfixed_userinfo}#{raw_host}:#{raw_port}#{prefixed_raw_path}#{prefixed_raw_query}" 348 | 349 | ::LogStash::Util::SafeURI.new(raw_url) 350 | end 351 | 352 | def template_exists?(name) 353 | response = @pool.head("/_template/#{name}") 354 | response.code >= 200 && response.code <= 299 355 | end 356 | 357 | def template_put(name, template) 358 | path = "/_template/#{name}" 359 | logger.info("Installing amazon_es template to #{path}") 360 | @pool.put(path, nil, LogStash::Json.dump(template)) 361 | end 362 | 363 | # Build a bulk item for an amazon_es update action 364 | def update_action_builder(args, source) 365 | if args[:_script] 366 | # Use the event as a hash from your script with variable name defined 367 | # by script_var_name (default: "event") 368 | # Ex: event["@timestamp"] 369 | source_orig = source 370 | source = { 'script' => {'params' => { @options[:script_var_name] => source_orig }} } 371 | if @options[:scripted_upsert] 372 | source['scripted_upsert'] = true 373 | source['upsert'] = {} 374 | elsif @options[:doc_as_upsert] 375 | source['upsert'] = source_orig 376 | else 377 | source['upsert'] = args.delete(:_upsert) if args[:_upsert] 378 | end 379 | case @options[:script_type] 380 | when 'indexed' 381 | source['script']['id'] = args.delete(:_script) 382 | when 'file' 383 | source['script']['file'] = args.delete(:_script) 384 | when 'inline' 385 | source['script']['inline'] = args.delete(:_script) 386 | end 387 | source['script']['lang'] = @options[:script_lang] if @options[:script_lang] != '' 388 | else 389 | source = { 'doc' => source } 390 | if @options[:doc_as_upsert] 391 | source['doc_as_upsert'] = true 392 | else 393 | source['upsert'] = args.delete(:_upsert) if args[:_upsert] 394 | end 395 | end 396 | [args, source] 397 | end 398 | end 399 | end end end 400 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/http_client/manticore_adapter.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require 'manticore' 6 | require 'cgi' 7 | require 'aws-sdk-core' 8 | require 'uri' 9 | 10 | module LogStash; module Outputs; class AmazonElasticSearch; class HttpClient; 11 | DEFAULT_HEADERS = { "content-type" => "application/json" } 12 | 13 | CredentialConfig = Struct.new( 14 | :access_key_id, 15 | :secret_access_key, 16 | :session_token, 17 | :profile, 18 | :instance_profile_credentials_retries, 19 | :instance_profile_credentials_timeout, 20 | :region) 21 | 22 | class ManticoreAdapter 23 | attr_reader :manticore, :logger 24 | 25 | def initialize(logger, options={}) 26 | @logger = logger 27 | options = options.clone || {} 28 | options[:ssl] = options[:ssl] || {} 29 | 30 | # We manage our own retries directly, so let's disable them here 31 | options[:automatic_retries] = 0 32 | # We definitely don't need cookies 33 | options[:cookies] = false 34 | 35 | @client_params = {:headers => DEFAULT_HEADERS.merge(options[:headers]|| {}),} 36 | 37 | @port = options[:port] || 9200 38 | @protocol = options[:protocol] || 'http' 39 | @region = options[:region] || 'us-east-1' 40 | @service_name = options[:service_name] || 'es' 41 | aws_access_key_id = options[:aws_access_key_id] || nil 42 | aws_secret_access_key = options[:aws_secret_access_key] || nil 43 | session_token = options[:session_token] || nil 44 | profile = options[:profile] || 'default' 45 | instance_cred_retries = options[:instance_profile_credentials_retries] || 0 46 | instance_cred_timeout = options[:instance_profile_credentials_timeout] || 1 47 | 48 | credential_config = CredentialConfig.new(aws_access_key_id, aws_secret_access_key, session_token, profile, instance_cred_retries, instance_cred_timeout, @region) 49 | @credentials = Aws::CredentialProviderChain.new(credential_config).resolve 50 | 51 | if options[:proxy] 52 | options[:proxy] = manticore_proxy_hash(options[:proxy]) 53 | end 54 | 55 | @manticore = ::Manticore::Client.new(options) 56 | end 57 | 58 | # Transform the proxy option to a hash. Manticore's support for non-hash 59 | # proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5 60 | # but this is not yet released 61 | def manticore_proxy_hash(proxy_uri) 62 | [:scheme, :port, :user, :password, :path].reduce(:host => proxy_uri.host) do |acc,opt| 63 | value = proxy_uri.send(opt) 64 | acc[opt] = value unless value.nil? || (value.is_a?(String) && value.empty?) 65 | acc 66 | end 67 | end 68 | 69 | def client 70 | @manticore 71 | end 72 | 73 | 74 | 75 | # Performs the request by invoking {Transport::Base#perform_request} with a block. 76 | # 77 | # @return [Response] 78 | # @see Transport::Base#perform_request 79 | # 80 | def perform_request(url, method, path, params={}, body=nil) 81 | # Perform 2-level deep merge on the params, so if the passed params and client params will both have hashes stored on a key they 82 | # will be merged as well, instead of choosing just one of the values 83 | params = (params || {}).merge(@client_params) { |key, oldval, newval| 84 | (oldval.is_a?(Hash) && newval.is_a?(Hash)) ? oldval.merge(newval) : newval 85 | } 86 | params[:headers] = params[:headers].clone 87 | 88 | 89 | params[:body] = body if body 90 | 91 | if url.user 92 | params[:auth] = { 93 | :user => CGI.unescape(url.user), 94 | # We have to unescape the password here since manticore won't do it 95 | # for us unless its part of the URL 96 | :password => CGI.unescape(url.password), 97 | :eager => true 98 | } 99 | end 100 | 101 | request_uri = format_url(url, path) 102 | 103 | if @protocol == "https" 104 | url = URI::HTTPS.build({:host=>URI(request_uri.to_s).host, :port=>@port.to_s, :path=>path}) 105 | else 106 | url = URI::HTTP.build({:host=>URI(request_uri.to_s).host, :port=>@port.to_s, :path=>path}) 107 | end 108 | 109 | 110 | request = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase, 111 | :headers => params[:headers],:body => params[:body]}) 112 | 113 | aws_signer = Aws::Sigv4::Signer.new(service: @service_name, region: @region, credentials_provider: @credentials) 114 | 115 | signed_key = aws_signer.sign_request( 116 | http_method: request.http_method, 117 | url: url, 118 | headers: params[:headers], 119 | body: params[:body] 120 | ) 121 | params[:headers] = params[:headers].merge(signed_key.headers) 122 | 123 | resp = @manticore.send(method.downcase, request_uri.to_s, params) 124 | 125 | # Manticore returns lazy responses by default 126 | # We want to block for our usage, this will wait for the repsonse 127 | # to finish 128 | resp.call 129 | # 404s are excluded because they are valid codes in the case of 130 | # template installation. We might need a better story around this later 131 | # but for our current purposes this is correct 132 | if resp.code < 200 || resp.code > 299 && resp.code != 404 133 | raise ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body, resp.body) 134 | end 135 | 136 | resp 137 | end 138 | 139 | def format_url(url, path_and_query=nil) 140 | request_uri = url.clone 141 | 142 | # We excise auth info from the URL in case manticore itself tries to stick 143 | # sensitive data in a thrown exception or log data 144 | request_uri.user = nil 145 | request_uri.password = nil 146 | 147 | return request_uri.to_s if path_and_query.nil? 148 | 149 | parsed_path_and_query = java.net.URI.new(path_and_query) 150 | 151 | query = request_uri.query 152 | parsed_query = parsed_path_and_query.query 153 | 154 | new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part| 155 | part && !part.empty? # Skip empty nil and "" 156 | end 157 | 158 | request_uri.query = new_query_parts.join("&") unless new_query_parts.empty? 159 | 160 | request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.path}".gsub(/\/{2,}/, "/") 161 | 162 | request_uri 163 | end 164 | 165 | def close 166 | @manticore.close 167 | end 168 | 169 | def host_unreachable_exceptions 170 | [::Manticore::Timeout,::Manticore::SocketException, ::Manticore::ClientProtocolException, ::Manticore::ResolutionFailure, Manticore::SocketTimeout] 171 | end 172 | end 173 | end; end; end; end 174 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/http_client/pool.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | module LogStash; module Outputs; class AmazonElasticSearch; class HttpClient; 6 | class Pool 7 | class NoConnectionAvailableError < Error; end 8 | class BadResponseCodeError < Error 9 | attr_reader :url, :response_code, :request_body, :response_body 10 | 11 | def initialize(response_code, url, request_body, response_body) 12 | @response_code = response_code 13 | @url = url 14 | @request_body = request_body 15 | @response_body = response_body 16 | end 17 | 18 | def message 19 | "Got response code '#{response_code}' contacting Elasticsearch at URL '#{@url}'" 20 | end 21 | end 22 | class HostUnreachableError < Error; 23 | attr_reader :original_error, :url 24 | 25 | def initialize(original_error, url) 26 | @original_error = original_error 27 | @url = url 28 | end 29 | 30 | def message 31 | "Elasticsearch Unreachable: [#{@url}][#{original_error.class}] #{original_error.message}" 32 | end 33 | end 34 | 35 | attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :sniffing_path, :bulk_path, :skip_healthcheck 36 | 37 | ROOT_URI_PATH = '/'.freeze 38 | 39 | DEFAULT_OPTIONS = { 40 | :healthcheck_path => ROOT_URI_PATH, 41 | :sniffing_path => "/_nodes/http", 42 | :bulk_path => "/_bulk", 43 | :scheme => 'http', 44 | :resurrect_delay => 5, 45 | :sniffing => false, 46 | :sniffer_delay => 10, 47 | :skip_healthcheck => false, 48 | }.freeze 49 | 50 | def initialize(logger, adapter, initial_urls=[], options={}) 51 | @logger = logger 52 | @adapter = adapter 53 | @metric = options[:metric] 54 | @initial_urls = initial_urls 55 | 56 | raise ArgumentError, "No URL Normalizer specified!" unless options[:url_normalizer] 57 | @url_normalizer = options[:url_normalizer] 58 | DEFAULT_OPTIONS.merge(options).tap do |merged| 59 | @bulk_path = merged[:bulk_path] 60 | @sniffing_path = merged[:sniffing_path] 61 | @healthcheck_path = merged[:healthcheck_path] 62 | @resurrect_delay = merged[:resurrect_delay] 63 | @sniffing = merged[:sniffing] 64 | @sniffer_delay = merged[:sniffer_delay] 65 | @skip_healthcheck = merged[:skip_healthcheck] 66 | end 67 | 68 | # Used for all concurrent operations in this class 69 | @state_mutex = Mutex.new 70 | 71 | # Holds metadata about all URLs 72 | @url_info = {} 73 | @stopping = false 74 | end 75 | 76 | def start 77 | update_urls(@initial_urls) 78 | start_resurrectionist 79 | start_sniffer if @sniffing 80 | end 81 | 82 | def close 83 | @state_mutex.synchronize { @stopping = true } 84 | 85 | logger.debug "Stopping sniffer" 86 | stop_sniffer 87 | 88 | logger.debug "Stopping resurrectionist" 89 | stop_resurrectionist 90 | 91 | logger.debug "Waiting for in use manticore connections" 92 | wait_for_in_use_connections 93 | 94 | logger.debug("Closing adapter #{@adapter}") 95 | @adapter.close 96 | end 97 | 98 | def wait_for_in_use_connections 99 | until in_use_connections.empty? 100 | logger.info "Blocked on shutdown to in use connections #{@state_mutex.synchronize {@url_info}}" 101 | sleep 1 102 | end 103 | end 104 | 105 | def in_use_connections 106 | @state_mutex.synchronize { @url_info.values.select {|v| v[:in_use] > 0 } } 107 | end 108 | 109 | def alive_urls_count 110 | @state_mutex.synchronize { @url_info.values.select {|v| !v[:state] == :alive }.count } 111 | end 112 | 113 | def url_info 114 | @state_mutex.synchronize { @url_info } 115 | end 116 | 117 | def maximum_seen_major_version 118 | @state_mutex.synchronize do 119 | @maximum_seen_major_version 120 | end 121 | end 122 | 123 | def urls 124 | url_info.keys 125 | end 126 | 127 | def until_stopped(task_name, delay) 128 | last_done = Time.now 129 | until @state_mutex.synchronize { @stopping } 130 | begin 131 | now = Time.now 132 | if (now - last_done) >= delay 133 | last_done = now 134 | yield 135 | end 136 | sleep 1 137 | rescue => e 138 | logger.warn( 139 | "Error while performing #{task_name}", 140 | :error_message => e.message, 141 | :class => e.class.name, 142 | :backtrace => e.backtrace 143 | ) 144 | end 145 | end 146 | end 147 | 148 | def start_sniffer 149 | @sniffer = Thread.new do 150 | until_stopped("sniffing", sniffer_delay) do 151 | begin 152 | sniff! 153 | rescue NoConnectionAvailableError => e 154 | @state_mutex.synchronize { # Synchronize around @url_info 155 | logger.warn("Elasticsearch output attempted to sniff for new connections but cannot. No living connections are detected. Pool contains the following current URLs", :url_info => @url_info) } 156 | end 157 | end 158 | end 159 | end 160 | 161 | # Sniffs the cluster then updates the internal URLs 162 | def sniff! 163 | update_urls(check_sniff) 164 | end 165 | 166 | ES1_SNIFF_RE_URL = /\[([^\/]*)?\/?([^:]*):([0-9]+)\]/ 167 | ES2_SNIFF_RE_URL = /([^\/]*)?\/?([^:]*):([0-9]+)/ 168 | # Sniffs and returns the results. Does not update internal URLs! 169 | def check_sniff 170 | _, url_meta, resp = perform_request(:get, @sniffing_path) 171 | @metric.increment(:sniff_requests) 172 | parsed = LogStash::Json.load(resp.body) 173 | nodes = parsed['nodes'] 174 | if !nodes || nodes.empty? 175 | @logger.warn("Sniff returned no nodes! Will not update hosts.") 176 | return nil 177 | else 178 | case major_version(url_meta[:version]) 179 | when 5, 6 180 | sniff_5x_and_above(nodes) 181 | when 2, 1 182 | sniff_2x_1x(nodes) 183 | else 184 | @logger.warn("Could not determine version for nodes in ES cluster!") 185 | return nil 186 | end 187 | end 188 | end 189 | 190 | def major_version(version_string) 191 | version_string.split('.').first.to_i 192 | end 193 | 194 | def sniff_5x_and_above(nodes) 195 | nodes.map do |id,info| 196 | # Skip master-only nodes 197 | next if info["roles"] && info["roles"] == ["master"] 198 | 199 | if info["http"] 200 | uri = LogStash::Util::SafeURI.new(info["http"]["publish_address"]) 201 | end 202 | end.compact 203 | end 204 | 205 | def sniff_2x_1x(nodes) 206 | nodes.map do |id,info| 207 | # TODO Make sure this works with shield. Does that listed 208 | # stuff as 'https_address?' 209 | 210 | addr_str = info['http_address'].to_s 211 | next unless addr_str # Skip hosts with HTTP disabled 212 | 213 | # Only connect to nodes that serve data 214 | # this will skip connecting to client, tribe, and master only nodes 215 | # Note that if 'attributes' is NOT set, then that's just a regular node 216 | # with master + data + client enabled, so we allow that 217 | attributes = info['attributes'] 218 | next if attributes && attributes['data'] == 'false' 219 | 220 | matches = addr_str.match(ES1_SNIFF_RE_URL) || addr_str.match(ES2_SNIFF_RE_URL) 221 | if matches 222 | host = matches[1].empty? ? matches[2] : matches[1] 223 | port = matches[3] 224 | ::LogStash::Util::SafeURI.new("#{host}:#{port}") 225 | end 226 | end.compact 227 | end 228 | 229 | def stop_sniffer 230 | @sniffer.join if @sniffer 231 | end 232 | 233 | def sniffer_alive? 234 | @sniffer ? @sniffer.alive? : nil 235 | end 236 | 237 | def start_resurrectionist 238 | @resurrectionist = Thread.new do 239 | until_stopped("resurrection", @resurrect_delay) do 240 | healthcheck! 241 | end 242 | end 243 | end 244 | 245 | def healthcheck! 246 | # Try to keep locking granularity low such that we don't affect IO... 247 | @state_mutex.synchronize { @url_info.select { |url, meta| meta[:state] != :alive } }.each do |url, meta| 248 | begin 249 | if skip_healthcheck == false 250 | logger.info("Running health check to see if an Elasticsearch connection is working", 251 | :healthcheck_url => url, :path => @healthcheck_path) 252 | response = perform_request_to_url(url, :head, @healthcheck_path) 253 | # If no exception was raised it must have succeeded! 254 | logger.warn("Restored connection to ES instance", :url => url.sanitized.to_s) 255 | # We reconnected to this node, check its ES version 256 | es_version = get_es_version(url) 257 | @state_mutex.synchronize do 258 | meta[:version] = es_version 259 | major = major_version(es_version) 260 | if !@maximum_seen_major_version 261 | @logger.info("ES Output version determined", :es_version => major) 262 | set_new_major_version(major) 263 | elsif major > @maximum_seen_major_version 264 | @logger.warn("Detected a node with a higher major version than previously observed. This could be the result of an amazon_es cluster upgrade.", :previous_major => @maximum_seen_major_version, :new_major => major, :node_url => url) 265 | set_new_major_version(major) 266 | end 267 | meta[:state] = :alive 268 | end 269 | elsif skip_healthcheck == true 270 | set_new_major_version(7) 271 | meta[:state] = :alive 272 | end 273 | rescue HostUnreachableError, BadResponseCodeError => e 274 | logger.warn("Attempted to resurrect connection to dead ES instance, but got an error.", url: url.sanitized.to_s, error_type: e.class, error: e.message) 275 | end 276 | end 277 | end 278 | 279 | def stop_resurrectionist 280 | @resurrectionist.join if @resurrectionist 281 | end 282 | 283 | def resurrectionist_alive? 284 | @resurrectionist ? @resurrectionist.alive? : nil 285 | end 286 | 287 | def perform_request(method, path, params={}, body=nil) 288 | with_connection do |url, url_meta| 289 | resp = perform_request_to_url(url, method, path, params, body) 290 | [url, url_meta, resp] 291 | end 292 | end 293 | 294 | [:get, :put, :post, :delete, :patch, :head].each do |method| 295 | define_method(method) do |path, params={}, body=nil| 296 | _, _, response = perform_request(method, path, params, body) 297 | response 298 | end 299 | end 300 | 301 | def perform_request_to_url(url, method, path, params={}, body=nil) 302 | res = @adapter.perform_request(url, method, path, params, body) 303 | rescue *@adapter.host_unreachable_exceptions => e 304 | raise HostUnreachableError.new(e, url), "Could not reach host #{e.class}: #{e.message}" 305 | end 306 | 307 | def normalize_url(uri) 308 | u = @url_normalizer.call(uri) 309 | if !u.is_a?(::LogStash::Util::SafeURI) 310 | raise "URL Normalizer returned a '#{u.class}' rather than a SafeURI! This shouldn't happen!" 311 | end 312 | u 313 | end 314 | 315 | def update_urls(new_urls) 316 | return if new_urls.nil? 317 | 318 | # Normalize URLs 319 | new_urls = new_urls.map(&method(:normalize_url)) 320 | 321 | # Used for logging nicely 322 | state_changes = {:removed => [], :added => []} 323 | @state_mutex.synchronize do 324 | # Add new connections 325 | new_urls.each do |url| 326 | # URI objects don't have real hash equality! So, since this isn't perf sensitive we do a linear scan 327 | unless @url_info.keys.include?(url) 328 | state_changes[:added] << url 329 | add_url(url) 330 | end 331 | end 332 | 333 | # Delete connections not in the new list 334 | @url_info.each do |url,_| 335 | unless new_urls.include?(url) 336 | state_changes[:removed] << url 337 | remove_url(url) 338 | end 339 | end 340 | end 341 | 342 | if state_changes[:removed].size > 0 || state_changes[:added].size > 0 343 | if logger.info? 344 | logger.info("Elasticsearch pool URLs updated", :changes => state_changes) 345 | end 346 | end 347 | 348 | # Run an inline healthcheck anytime URLs are updated 349 | # This guarantees that during startup / post-startup 350 | # sniffing we don't have idle periods waiting for the 351 | # periodic sniffer to allow new hosts to come online 352 | healthcheck! 353 | end 354 | 355 | def size 356 | @state_mutex.synchronize { @url_info.size } 357 | end 358 | 359 | def es_versions 360 | @state_mutex.synchronize { @url_info.size } 361 | end 362 | 363 | def add_url(url) 364 | @url_info[url] ||= empty_url_meta 365 | end 366 | 367 | def remove_url(url) 368 | @url_info.delete(url) 369 | end 370 | 371 | def empty_url_meta 372 | { 373 | :in_use => 0, 374 | :state => :unknown 375 | } 376 | end 377 | 378 | def with_connection 379 | url, url_meta = get_connection 380 | 381 | # Custom error class used here so that users may retry attempts if they receive this error 382 | # should they choose to 383 | raise NoConnectionAvailableError, "No Available connections" unless url 384 | yield url, url_meta 385 | rescue HostUnreachableError => e 386 | # Mark the connection as dead here since this is likely not transient 387 | mark_dead(url, e) 388 | raise e 389 | rescue BadResponseCodeError => e 390 | # These aren't discarded from the pool because these are often very transient 391 | # errors 392 | raise e 393 | rescue => e 394 | logger.warn("UNEXPECTED POOL ERROR", :e => e) 395 | raise e 396 | ensure 397 | return_connection(url) 398 | end 399 | 400 | def mark_dead(url, error) 401 | @state_mutex.synchronize do 402 | meta = @url_info[url] 403 | # In case a sniff happened removing the metadata just before there's nothing to mark 404 | # This is an extreme edge case, but it can happen! 405 | return unless meta 406 | logger.warn("Marking url as dead. Last error: [#{error.class}] #{error.message}", 407 | :url => url, :error_message => error.message, :error_class => error.class.name) 408 | meta[:state] = :dead 409 | meta[:last_error] = error 410 | meta[:last_errored_at] = Time.now 411 | end 412 | end 413 | 414 | def url_meta(url) 415 | @state_mutex.synchronize do 416 | @url_info[url] 417 | end 418 | end 419 | 420 | def get_connection 421 | @state_mutex.synchronize do 422 | # The goal here is to pick a random connection from the least-in-use connections 423 | # We want some randomness so that we don't hit the same node over and over, but 424 | # we also want more 'fair' behavior in the event of high concurrency 425 | eligible_set = nil 426 | lowest_value_seen = nil 427 | @url_info.each do |url,meta| 428 | meta_in_use = meta[:in_use] 429 | next if meta[:state] == :dead 430 | 431 | if lowest_value_seen.nil? || meta_in_use < lowest_value_seen 432 | lowest_value_seen = meta_in_use 433 | eligible_set = [[url, meta]] 434 | elsif lowest_value_seen == meta_in_use 435 | eligible_set << [url, meta] 436 | end 437 | end 438 | 439 | return nil if eligible_set.nil? 440 | 441 | pick, pick_meta = eligible_set.sample 442 | pick_meta[:in_use] += 1 443 | 444 | [pick, pick_meta] 445 | end 446 | end 447 | 448 | def return_connection(url) 449 | @state_mutex.synchronize do 450 | if @url_info[url] # Guard against the condition where the connection has already been deleted 451 | @url_info[url][:in_use] -= 1 452 | end 453 | end 454 | end 455 | 456 | def get_es_version(url) 457 | request = perform_request_to_url(url, :get, ROOT_URI_PATH) 458 | LogStash::Json.load(request.body)["version"]["number"] 459 | end 460 | 461 | def set_new_major_version(version) 462 | @maximum_seen_major_version = version 463 | if @maximum_seen_major_version >= 6 464 | @logger.warn("Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type", :es_version => @maximum_seen_major_version) 465 | end 466 | end 467 | end 468 | end; end; end; end; 469 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/http_client_builder.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require 'cgi' 6 | 7 | module LogStash; module Outputs; class AmazonElasticSearch; 8 | module HttpClientBuilder 9 | def self.build(logger, hosts, params) 10 | client_settings = { 11 | :pool_max => params["pool_max"], 12 | :pool_max_per_route => params["pool_max_per_route"], 13 | :check_connection_timeout => params["validate_after_inactivity"], 14 | :http_compression => params["http_compression"], 15 | :headers => params["custom_headers"] 16 | } 17 | 18 | client_settings[:proxy] = params["proxy"] if params["proxy"] 19 | 20 | common_options = { 21 | :client_settings => client_settings, 22 | :metric => params["metric"], 23 | :resurrect_delay => params["resurrect_delay"] 24 | } 25 | 26 | if params["sniffing"] 27 | common_options[:sniffing] = true 28 | common_options[:sniffer_delay] = params["sniffing_delay"] 29 | end 30 | 31 | common_options[:timeout] = params["timeout"] if params["timeout"] 32 | 33 | if params["path"] 34 | client_settings[:path] = dedup_slashes("/#{params["path"]}/") 35 | end 36 | 37 | common_options[:bulk_path] = if params["bulk_path"] 38 | dedup_slashes("/#{params["bulk_path"]}") 39 | else 40 | dedup_slashes("/#{params["path"]}/_bulk") 41 | end 42 | 43 | common_options[:sniffing_path] = if params["sniffing_path"] 44 | dedup_slashes("/#{params["sniffing_path"]}") 45 | else 46 | dedup_slashes("/#{params["path"]}/_nodes/http") 47 | end 48 | 49 | common_options[:healthcheck_path] = if params["healthcheck_path"] 50 | dedup_slashes("/#{params["healthcheck_path"]}") 51 | else 52 | dedup_slashes("/#{params["path"]}") 53 | end 54 | 55 | if params["parameters"] 56 | client_settings[:parameters] = params["parameters"] 57 | end 58 | 59 | logger.debug? && logger.debug("Normalizing http path", :path => params["path"], :normalized => client_settings[:path]) 60 | 61 | client_settings.merge! setup_ssl(logger, params) 62 | common_options.merge! setup_basic_auth(logger, params) 63 | 64 | external_version_types = ["external", "external_gt", "external_gte"] 65 | # External Version validation 66 | raise( 67 | LogStash::ConfigurationError, 68 | "External versioning requires the presence of a version number." 69 | ) if external_version_types.include?(params.fetch('version_type', '')) and params.fetch("version", nil) == nil 70 | 71 | 72 | # Create API setup 73 | raise( 74 | LogStash::ConfigurationError, 75 | "External versioning is not supported by the create action." 76 | ) if params['action'] == 'create' and external_version_types.include?(params.fetch('version_type', '')) 77 | 78 | # Update API setup 79 | raise( LogStash::ConfigurationError, 80 | "doc_as_upsert and scripted_upsert are mutually exclusive." 81 | ) if params["doc_as_upsert"] and params["scripted_upsert"] 82 | 83 | raise( 84 | LogStash::ConfigurationError, 85 | "Specifying action => 'update' needs a document_id." 86 | ) if params['action'] == 'update' and params.fetch('document_id', '') == '' 87 | 88 | raise( 89 | LogStash::ConfigurationError, 90 | "External versioning is not supported by the update action. See https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html." 91 | ) if params['action'] == 'update' and external_version_types.include?(params.fetch('version_type', '')) 92 | 93 | # Update API setup 94 | update_options = { 95 | :doc_as_upsert => params["doc_as_upsert"], 96 | :script_var_name => params["script_var_name"], 97 | :script_type => params["script_type"], 98 | :script_lang => params["script_lang"], 99 | :scripted_upsert => params["scripted_upsert"] 100 | } 101 | common_options.merge! update_options if params["action"] == 'update' 102 | create_http_client(common_options.merge(:hosts => hosts, 103 | :logger => logger, 104 | :protocol => params["protocol"], 105 | :port => params["port"], 106 | :region => params["region"], 107 | :service_name => params["service_name"], 108 | :aws_access_key_id => params["aws_access_key_id"], 109 | :aws_secret_access_key => params["aws_secret_access_key"], 110 | :max_bulk_bytes => params["max_bulk_bytes"], 111 | :skip_healthcheck => params["skip_healthcheck"]) 112 | ) 113 | end 114 | 115 | def self.create_http_client(options) 116 | LogStash::Outputs::AmazonElasticSearch::HttpClient.new(options) 117 | end 118 | 119 | def self.setup_ssl(logger, params) 120 | params["ssl"] = true if params["hosts"].any? {|h| h.scheme == "https" } 121 | return {} if params["ssl"].nil? 122 | 123 | return {:ssl => {:enabled => false}} if params["ssl"] == false 124 | 125 | cacert, truststore, truststore_password, keystore, keystore_password = 126 | params.values_at('cacert', 'truststore', 'truststore_password', 'keystore', 'keystore_password') 127 | 128 | if cacert && truststore 129 | raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if truststore 130 | end 131 | 132 | ssl_options = {:enabled => true} 133 | 134 | if cacert 135 | ssl_options[:ca_file] = cacert 136 | elsif truststore 137 | ssl_options[:truststore_password] = truststore_password.value if truststore_password 138 | end 139 | 140 | ssl_options[:truststore] = truststore if truststore 141 | if keystore 142 | ssl_options[:keystore] = keystore 143 | ssl_options[:keystore_password] = keystore_password.value if keystore_password 144 | end 145 | if !params["ssl_certificate_verification"] 146 | logger.warn [ 147 | "** WARNING ** Detected UNSAFE options in amazon_es output configuration!", 148 | "** WARNING ** You have enabled encryption but DISABLED certificate verification.", 149 | "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true" 150 | ].join("\n") 151 | ssl_options[:verify] = false 152 | end 153 | { ssl: ssl_options } 154 | end 155 | 156 | def self.setup_basic_auth(logger, params) 157 | user, password = params["user"], params["password"] 158 | 159 | return {} unless user && password && password.value 160 | 161 | { 162 | :user => CGI.escape(user), 163 | :password => CGI.escape(password.value) 164 | } 165 | end 166 | 167 | private 168 | def self.dedup_slashes(url) 169 | url.gsub(/\/+/, "/") 170 | end 171 | end 172 | end; end; end 173 | -------------------------------------------------------------------------------- /lib/logstash/outputs/amazon_es/template_manager.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | module LogStash; module Outputs; class AmazonElasticSearch 6 | class TemplateManager 7 | # To be mixed into the amazon_es plugin base 8 | def self.install_template(plugin) 9 | return unless plugin.manage_template 10 | plugin.logger.info("Using mapping template from", :path => plugin.template) 11 | template = get_template(plugin.template, plugin.maximum_seen_major_version) 12 | plugin.logger.info("Attempting to install template", :manage_template => template) 13 | install(plugin.client, plugin.template_name, template, plugin.template_overwrite) 14 | rescue => e 15 | plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace) 16 | end 17 | 18 | private 19 | def self.get_template(path, es_major_version) 20 | template_path = path || default_template_path(es_major_version) 21 | read_template_file(template_path) 22 | end 23 | 24 | def self.install(client, template_name, template, template_overwrite) 25 | client.template_install(template_name, template, template_overwrite) 26 | end 27 | 28 | def self.default_template_path(es_major_version) 29 | template_version = es_major_version == 1 ? 2 : es_major_version 30 | default_template_name = "elasticsearch-template-es#{template_version}x.json" 31 | ::File.expand_path(default_template_name, ::File.dirname(__FILE__)) 32 | end 33 | 34 | def self.read_template_file(template_path) 35 | raise ArgumentError, "Template file '#{@template_path}' could not be found!" unless ::File.exists?(template_path) 36 | template_data = ::IO.read(template_path) 37 | LogStash::Json.load(template_data) 38 | end 39 | end 40 | end end end 41 | -------------------------------------------------------------------------------- /logstash-output-amazon_es.gemspec: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Gem::Specification.new do |s| 6 | s.name = 'logstash-output-amazon_es' 7 | s.version = '8.0.0' 8 | s.licenses = ['Apache-2.0'] 9 | s.summary = "Logstash Output to Amazon Elasticsearch Service" 10 | s.description = "Output events to Amazon Elasticsearch Service with V4 signing" 11 | s.authors = ["Amazon"] 12 | s.email = 'feedback-prod-elasticsearch@amazon.com' 13 | s.homepage = "https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/index.html" 14 | s.require_paths = ["lib"] 15 | 16 | s.platform = RUBY_PLATFORM 17 | 18 | # Files 19 | s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] 20 | 21 | # Tests 22 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 23 | 24 | # Special flag to let us know this is actually a logstash plugin 25 | s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } 26 | 27 | s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0' 28 | s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0'] 29 | s.add_runtime_dependency 'cabin', ['~> 0.6'] 30 | s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" 31 | s.add_runtime_dependency 'aws-sdk', '~> 3' 32 | 33 | s.add_development_dependency 'logstash-codec-plain' 34 | s.add_development_dependency 'logstash-devutils', "~> 1.3", ">= 1.3.1" 35 | s.add_development_dependency 'flores', '~> 0' 36 | end 37 | -------------------------------------------------------------------------------- /spec/es_spec_helper.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/devutils/rspec/spec_helper" 6 | require 'manticore' 7 | 8 | # by default exclude secure_integration tests unless requested 9 | # normal integration specs are already excluded by devutils' spec helper 10 | RSpec.configure do |config| 11 | config.filter_run_excluding config.exclusion_filter.add(:secure_integration => true) 12 | end 13 | 14 | module ESHelper 15 | def get_host_port 16 | "127.0.0.1" 17 | end 18 | 19 | def self.es_version 20 | RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] 21 | end 22 | 23 | def self.es_version_satisfies?(*requirement) 24 | es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] 25 | if es_version.nil? 26 | puts "Info: ES_VERSION environment or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call." 27 | return false 28 | end 29 | es_release_version = Gem::Version.new(es_version).release 30 | Gem::Requirement.new(requirement).satisfied_by?(es_release_version) 31 | end 32 | end 33 | 34 | RSpec.configure do |config| 35 | config.include ESHelper 36 | end 37 | -------------------------------------------------------------------------------- /spec/unit/http_client_builder_spec.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/devutils/rspec/spec_helper" 6 | require "logstash/outputs/amazon_es" 7 | require "logstash/outputs/amazon_es/http_client" 8 | require "logstash/outputs/amazon_es/http_client_builder" 9 | 10 | describe LogStash::Outputs::AmazonElasticSearch::HttpClientBuilder do 11 | describe "auth setup with url encodable passwords" do 12 | let(:klass) { LogStash::Outputs::AmazonElasticSearch::HttpClientBuilder } 13 | let(:user) { "foo@bar"} 14 | let(:password) {"baz@blah" } 15 | let(:password_secured) do 16 | secured = double("password") 17 | allow(secured).to receive(:value).and_return(password) 18 | secured 19 | end 20 | let(:options) { {"user" => user, "password" => password} } 21 | let(:logger) { mock("logger") } 22 | let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) } 23 | 24 | it "should return the user escaped" do 25 | expect(auth_setup[:user]).to eql(CGI.escape(user)) 26 | end 27 | 28 | it "should return the password escaped" do 29 | expect(auth_setup[:password]).to eql(CGI.escape(password)) 30 | end 31 | end 32 | 33 | describe "customizing action paths" do 34 | let(:hosts) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200") ] } 35 | let(:options) { {"hosts" => hosts , 36 | "protocol" => "http", 37 | "port" => 9200, 38 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 39 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 40 | let(:logger) { double("logger") } 41 | before :each do 42 | [:debug, :debug?, :info?, :info, :warn].each do |level| 43 | allow(logger).to receive(level) 44 | end 45 | end 46 | 47 | describe "healthcheck_path" do 48 | 49 | context "when setting bulk_path" do 50 | let(:bulk_path) { "/meh" } 51 | let(:options) { super.merge("bulk_path" => bulk_path) } 52 | 53 | context "when using path" do 54 | let(:options) { super.merge("path" => "/path") } 55 | it "ignores the path setting" do 56 | expect(described_class).to receive(:create_http_client) do |options| 57 | expect(options[:bulk_path]).to eq(bulk_path) 58 | end 59 | described_class.build(logger, hosts, options) 60 | end 61 | end 62 | context "when not using path" do 63 | 64 | it "uses the bulk_path setting" do 65 | expect(described_class).to receive(:create_http_client) do |options| 66 | expect(options[:bulk_path]).to eq(bulk_path) 67 | end 68 | described_class.build(logger, hosts, options) 69 | end 70 | end 71 | end 72 | 73 | context "when not setting bulk_path" do 74 | 75 | context "when using path" do 76 | let(:path) { "/meh" } 77 | let(:options) { super.merge("path" => path) } 78 | it "sets bulk_path to path+_bulk" do 79 | expect(described_class).to receive(:create_http_client) do |options| 80 | expect(options[:bulk_path]).to eq("#{path}/_bulk") 81 | end 82 | described_class.build(logger, hosts, options) 83 | end 84 | end 85 | 86 | context "when not using path" do 87 | it "sets the bulk_path to _bulk" do 88 | expect(described_class).to receive(:create_http_client) do |options| 89 | expect(options[:bulk_path]).to eq("/_bulk") 90 | end 91 | described_class.build(logger, hosts, options) 92 | end 93 | end 94 | end 95 | end 96 | describe "healthcheck_path" do 97 | context "when setting healthcheck_path" do 98 | let(:healthcheck_path) { "/meh" } 99 | let(:options) { super.merge("healthcheck_path" => healthcheck_path) } 100 | 101 | context "when using path" do 102 | let(:options) { super.merge("path" => "/path") } 103 | it "ignores the path setting" do 104 | expect(described_class).to receive(:create_http_client) do |options| 105 | expect(options[:healthcheck_path]).to eq(healthcheck_path) 106 | end 107 | described_class.build(logger, hosts, options) 108 | end 109 | end 110 | context "when not using path" do 111 | 112 | it "uses the healthcheck_path setting" do 113 | expect(described_class).to receive(:create_http_client) do |options| 114 | expect(options[:healthcheck_path]).to eq(healthcheck_path) 115 | end 116 | described_class.build(logger, hosts, options) 117 | end 118 | end 119 | end 120 | 121 | context "when not setting healthcheck_path" do 122 | 123 | context "when using path" do 124 | let(:path) { "/meh" } 125 | let(:options) { super.merge("path" => path) } 126 | it "sets healthcheck_path to path" do 127 | expect(described_class).to receive(:create_http_client) do |options| 128 | expect(options[:healthcheck_path]).to eq(path) 129 | end 130 | described_class.build(logger, hosts, options) 131 | end 132 | end 133 | 134 | context "when not using path" do 135 | it "sets the healthcheck_path to root" do 136 | expect(described_class).to receive(:create_http_client) do |options| 137 | expect(options[:healthcheck_path]).to eq("/") 138 | end 139 | described_class.build(logger, hosts, options) 140 | end 141 | end 142 | end 143 | end 144 | describe "sniffing_path" do 145 | context "when setting sniffing_path" do 146 | let(:sniffing_path) { "/meh" } 147 | let(:options) { super.merge("sniffing_path" => sniffing_path) } 148 | 149 | context "when using path" do 150 | let(:options) { super.merge("path" => "/path") } 151 | it "ignores the path setting" do 152 | expect(described_class).to receive(:create_http_client) do |options| 153 | expect(options[:sniffing_path]).to eq(sniffing_path) 154 | end 155 | described_class.build(logger, hosts, options) 156 | end 157 | end 158 | context "when not using path" do 159 | 160 | it "uses the sniffing_path setting" do 161 | expect(described_class).to receive(:create_http_client) do |options| 162 | expect(options[:sniffing_path]).to eq(sniffing_path) 163 | end 164 | described_class.build(logger, hosts, options) 165 | end 166 | end 167 | end 168 | 169 | context "when not setting sniffing_path" do 170 | 171 | context "when using path" do 172 | let(:path) { "/meh" } 173 | let(:options) { super.merge("path" => path) } 174 | it "sets sniffing_path to path+_nodes/http" do 175 | expect(described_class).to receive(:create_http_client) do |options| 176 | expect(options[:sniffing_path]).to eq("#{path}/_nodes/http") 177 | end 178 | described_class.build(logger, hosts, options) 179 | end 180 | end 181 | 182 | context "when not using path" do 183 | it "sets the sniffing_path to _nodes/http" do 184 | expect(described_class).to receive(:create_http_client) do |options| 185 | expect(options[:sniffing_path]).to eq("/_nodes/http") 186 | end 187 | described_class.build(logger, hosts, options) 188 | end 189 | end 190 | end 191 | end 192 | end 193 | end 194 | -------------------------------------------------------------------------------- /spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/devutils/rspec/spec_helper" 6 | require "logstash/outputs/amazon_es/http_client" 7 | 8 | describe LogStash::Outputs::AmazonElasticSearch::HttpClient::ManticoreAdapter do 9 | let(:logger) { Cabin::Channel.get } 10 | let(:options) { {:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA', 11 | :aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'} } 12 | 13 | subject { described_class.new(logger, options) } 14 | 15 | it "should raise an exception if requests are issued after close" do 16 | subject.close 17 | expect { subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/') }.to raise_error(::Manticore::ClientStoppedException) 18 | end 19 | 20 | it "should implement host unreachable exceptions" do 21 | expect(subject.host_unreachable_exceptions).to be_a(Array) 22 | end 23 | 24 | 25 | describe "bad response codes" do 26 | let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") } 27 | 28 | it "should raise a bad response code error" do 29 | resp = double("response") 30 | allow(resp).to receive(:call) 31 | allow(resp).to receive(:code).and_return(500) 32 | allow(resp).to receive(:body).and_return("a body") 33 | 34 | expect(subject.manticore).to receive(:get). 35 | with(uri.to_s + "/", anything). 36 | and_return(resp) 37 | 38 | uri_with_path = uri.clone 39 | uri_with_path.path = "/" 40 | 41 | expect(::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new). 42 | with(resp.code, uri_with_path, nil, resp.body).and_call_original 43 | 44 | expect do 45 | subject.perform_request(uri, :get, "/") 46 | end.to raise_error(::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError) 47 | end 48 | end 49 | 50 | describe "format_url" do 51 | let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") } 52 | let(:path) { "_bulk" } 53 | subject { described_class.new(double("logger"), 54 | {:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA', 55 | :aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'} ) } 56 | 57 | it "should add the path argument to the uri's path" do 58 | expect(subject.format_url(url, path).path).to eq("/path/_bulk") 59 | end 60 | 61 | context "when uri contains query parameters" do 62 | let(:query_params) { "query=value&key=value2" } 63 | let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") } 64 | let(:formatted) { subject.format_url(url, path)} 65 | 66 | 67 | it "should retain query_params after format" do 68 | expect(formatted.query).to eq(query_params) 69 | end 70 | 71 | context "and the path contains query parameters" do 72 | let(:path) { "/special_path?specialParam=123" } 73 | 74 | it "should join the query correctly" do 75 | expect(formatted.query).to eq(query_params + "&specialParam=123") 76 | end 77 | end 78 | end 79 | 80 | context "when the path contains query parameters" do 81 | let(:path) { "/special_bulk?pathParam=1"} 82 | let(:formatted) { subject.format_url(url, path) } 83 | 84 | it "should add the path correctly" do 85 | expect(formatted.path).to eq("#{url.path}special_bulk") 86 | end 87 | 88 | it "should add the query parameters correctly" do 89 | expect(formatted.query).to eq("pathParam=1") 90 | end 91 | end 92 | 93 | context "when uri contains credentials" do 94 | let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") } 95 | let(:formatted) { subject.format_url(url, path) } 96 | 97 | it "should remove credentials after format" do 98 | expect(formatted.userinfo).to be_nil 99 | end 100 | end 101 | end 102 | 103 | describe "integration specs", :integration => true do 104 | it "should perform correct tests without error" do 105 | resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/") 106 | expect(resp.code).to eql(200) 107 | end 108 | end 109 | end 110 | -------------------------------------------------------------------------------- /spec/unit/outputs/elasticsearch/http_client/pool_spec.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/devutils/rspec/spec_helper" 6 | require "logstash/outputs/amazon_es/http_client" 7 | require "json" 8 | 9 | describe LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool do 10 | let(:logger) { Cabin::Channel.get } 11 | let(:adapter) { LogStash::Outputs::AmazonElasticSearch::HttpClient::ManticoreAdapter.new(logger, 12 | {:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA', 13 | :aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'}) } 14 | let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] } 15 | let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests 16 | let(:es_node_versions) { [ "0.0.0" ] } 17 | 18 | subject { described_class.new(logger, adapter, initial_urls, options) } 19 | 20 | let(:manticore_double) { double("manticore a") } 21 | before(:each) do 22 | 23 | response_double = double("manticore response").as_null_object 24 | # Allow healtchecks 25 | allow(manticore_double).to receive(:head).with(any_args).and_return(response_double) 26 | allow(manticore_double).to receive(:get).with(any_args).and_return(response_double) 27 | allow(manticore_double).to receive(:close) 28 | 29 | allow(::Manticore::Client).to receive(:new).and_return(manticore_double) 30 | 31 | allow(subject).to receive(:get_es_version).with(any_args).and_return(*es_node_versions) 32 | end 33 | 34 | after do 35 | subject.close 36 | end 37 | 38 | describe "initialization" do 39 | it "should be successful" do 40 | expect { subject }.not_to raise_error 41 | subject.start 42 | end 43 | end 44 | 45 | describe "the resurrectionist" do 46 | before(:each) { subject.start } 47 | it "should start the resurrectionist when created" do 48 | expect(subject.resurrectionist_alive?).to eql(true) 49 | end 50 | 51 | it "should attempt to resurrect connections after the ressurrect delay" do 52 | expect(subject).to receive(:healthcheck!).once 53 | sleep(subject.resurrect_delay + 1) 54 | end 55 | 56 | describe "healthcheck url handling" do 57 | let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] } 58 | 59 | context "and not setting healthcheck_path" do 60 | it "performs the healthcheck to the root" do 61 | expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _| 62 | expect(method).to eq(:head) 63 | expect(url.path).to be_empty 64 | expect(req_path).to eq("/") 65 | end 66 | subject.healthcheck! 67 | end 68 | end 69 | 70 | context "and setting healthcheck_path" do 71 | let(:healthcheck_path) { "/my/health" } 72 | let(:options) { super.merge(:healthcheck_path => healthcheck_path) } 73 | it "performs the healthcheck to the healthcheck_path" do 74 | expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _| 75 | expect(method).to eq(:head) 76 | expect(url.path).to be_empty 77 | expect(req_path).to eq(healthcheck_path) 78 | end 79 | subject.healthcheck! 80 | end 81 | end 82 | end 83 | end 84 | 85 | describe "the sniffer" do 86 | before(:each) { subject.start } 87 | it "should not start the sniffer by default" do 88 | expect(subject.sniffer_alive?).to eql(nil) 89 | end 90 | 91 | context "when enabled" do 92 | let(:options) { super.merge(:sniffing => true)} 93 | 94 | it "should start the sniffer" do 95 | expect(subject.sniffer_alive?).to eql(true) 96 | end 97 | end 98 | end 99 | 100 | describe "closing" do 101 | before do 102 | subject.start 103 | # Simulate a single in use connection on the first check of this 104 | allow(adapter).to receive(:close).and_call_original 105 | allow(subject).to receive(:wait_for_in_use_connections).and_call_original 106 | allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[]) 107 | allow(subject).to receive(:start) 108 | subject.close 109 | end 110 | 111 | it "should close the adapter" do 112 | expect(adapter).to have_received(:close) 113 | end 114 | 115 | it "should stop the resurrectionist" do 116 | expect(subject.resurrectionist_alive?).to eql(false) 117 | end 118 | 119 | it "should stop the sniffer" do 120 | # If no sniffer (the default) returns nil 121 | expect(subject.sniffer_alive?).to be_falsey 122 | end 123 | 124 | it "should wait for in use connections to terminate" do 125 | expect(subject).to have_received(:wait_for_in_use_connections).once 126 | expect(subject).to have_received(:in_use_connections).twice 127 | end 128 | end 129 | 130 | describe "connection management" do 131 | before(:each) { subject.start } 132 | context "with only one URL in the list" do 133 | it "should use the only URL in 'with_connection'" do 134 | subject.with_connection do |c| 135 | expect(c).to eq(initial_urls.first) 136 | end 137 | end 138 | end 139 | 140 | context "with multiple URLs in the list" do 141 | before :each do 142 | allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil) 143 | end 144 | let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200") ] } 145 | 146 | it "should minimize the number of connections to a single URL" do 147 | connected_urls = [] 148 | 149 | # If we make 2x the number requests as we have URLs we should 150 | # connect to each URL exactly 2 times 151 | (initial_urls.size*2).times do 152 | u, meta = subject.get_connection 153 | connected_urls << u 154 | end 155 | 156 | connected_urls.each {|u| subject.return_connection(u) } 157 | initial_urls.each do |url| 158 | conn_count = connected_urls.select {|u| u == url}.size 159 | expect(conn_count).to eql(2) 160 | end 161 | end 162 | 163 | it "should correctly resurrect the dead" do 164 | u,m = subject.get_connection 165 | 166 | # The resurrectionist will call this to check on the backend 167 | response = double("response") 168 | expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response) 169 | 170 | subject.return_connection(u) 171 | subject.mark_dead(u, Exception.new) 172 | 173 | expect(subject.url_meta(u)[:state]).to eql(:dead) 174 | sleep subject.resurrect_delay + 1 175 | expect(subject.url_meta(u)[:state]).to eql(:alive) 176 | end 177 | end 178 | end 179 | 180 | describe "version tracking" do 181 | let(:initial_urls) { [ 182 | ::LogStash::Util::SafeURI.new("http://somehost:9200"), 183 | ::LogStash::Util::SafeURI.new("http://otherhost:9201") 184 | ] } 185 | 186 | before(:each) do 187 | allow(subject).to receive(:perform_request_to_url).and_return(nil) 188 | subject.start 189 | end 190 | 191 | it "picks the largest major version" do 192 | expect(subject.maximum_seen_major_version).to eq(0) 193 | end 194 | 195 | context "if there are nodes with multiple major versions" do 196 | let(:es_node_versions) { [ "0.0.0", "6.0.0" ] } 197 | it "picks the largest major version" do 198 | expect(subject.maximum_seen_major_version).to eq(6) 199 | end 200 | end 201 | end 202 | end 203 | -------------------------------------------------------------------------------- /spec/unit/outputs/elasticsearch/http_client_spec.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/devutils/rspec/spec_helper" 6 | require "logstash/outputs/amazon_es/http_client" 7 | require "java" 8 | 9 | describe LogStash::Outputs::AmazonElasticSearch::HttpClient do 10 | let(:ssl) { nil } 11 | let(:base_options) do 12 | opts = { 13 | :hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")], 14 | :logger => Cabin::Channel.get, 15 | :metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy), 16 | :protocol => "http", 17 | :port => 9200, 18 | :aws_access_key_id => "AAAAAAAAAAAAAAAAAAAA", 19 | :aws_secret_access_key => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", 20 | :max_bulk_bytes => 20 * 1024 * 1024 21 | } 22 | 23 | if !ssl.nil? # Shortcut to set this 24 | opts[:client_settings] = {:ssl => {:enabled => ssl}} 25 | end 26 | 27 | opts 28 | end 29 | 30 | describe "Host/URL Parsing" do 31 | subject { described_class.new(base_options) } 32 | 33 | let(:true_hostname) { "my-dash.hostname" } 34 | let(:ipv6_hostname) { "[::1]" } 35 | let(:ipv4_hostname) { "127.0.0.1" } 36 | let(:port) { 9200 } 37 | let(:max_bulk_bytes) { 20 * 1024 * 1024 } 38 | let(:protocol) {"http"} 39 | let(:aws_access_key_id) {"AAAAAAAAAAAAAAAAAAAA"} 40 | let(:aws_secret_access_key) {"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} 41 | let(:hostname_port) { "#{hostname}:#{port}" } 42 | let(:hostname_port_uri) { ::LogStash::Util::SafeURI.new("//#{hostname_port}") } 43 | let(:http_hostname_port) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}") } 44 | let(:https_hostname_port) { ::LogStash::Util::SafeURI.new("https://#{hostname_port}") } 45 | let(:http_hostname_port_path) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}/path") } 46 | 47 | shared_examples("proper host handling") do 48 | it "should properly transform a host:port string to a URL" do 49 | expect(subject.host_to_url(hostname_port_uri).to_s).to eq(http_hostname_port.to_s + "/") 50 | end 51 | 52 | it "should not raise an error with a / for a path" do 53 | expect(subject.host_to_url(::LogStash::Util::SafeURI.new("#{http_hostname_port}/"))).to eq(LogStash::Util::SafeURI.new("#{http_hostname_port}/")) 54 | end 55 | 56 | it "should parse full URLs correctly" do 57 | expect(subject.host_to_url(http_hostname_port).to_s).to eq(http_hostname_port.to_s + "/") 58 | end 59 | 60 | 61 | describe "path" do 62 | let(:url) { http_hostname_port_path } 63 | let(:base_options) { super.merge(:hosts => [url]) } 64 | 65 | it "should allow paths in a url" do 66 | expect(subject.host_to_url(url)).to eq(url) 67 | end 68 | 69 | context "with the path option set" do 70 | let(:base_options) { super.merge(:client_settings => {:path => "/otherpath"}) } 71 | 72 | it "should not allow paths in two places" do 73 | expect { 74 | subject.host_to_url(url) 75 | }.to raise_error(LogStash::ConfigurationError) 76 | end 77 | end 78 | 79 | context "with a path missing a leading /" do 80 | let(:url) { http_hostname_port } 81 | let(:base_options) { super.merge(:client_settings => {:path => "otherpath"}) } 82 | 83 | 84 | it "should automatically insert a / in front of path overlays" do 85 | expected = url.clone 86 | expected.path = url.path + "/otherpath" 87 | expect(subject.host_to_url(url)).to eq(expected) 88 | end 89 | end 90 | end 91 | end 92 | 93 | describe "an regular hostname" do 94 | let(:hostname) { true_hostname } 95 | include_examples("proper host handling") 96 | end 97 | 98 | describe "an ipv4 host" do 99 | let(:hostname) { ipv4_hostname } 100 | include_examples("proper host handling") 101 | end 102 | 103 | end 104 | 105 | describe "get" do 106 | subject { described_class.new(base_options) } 107 | let(:body) { "foobar" } 108 | let(:path) { "/hello-id" } 109 | let(:get_response) { 110 | double("response", :body => LogStash::Json::dump( { "body" => body })) 111 | } 112 | 113 | it "returns the hash response" do 114 | expect(subject.pool).to receive(:get).with(path, nil).and_return(get_response) 115 | expect(subject.get(path)["body"]).to eq(body) 116 | end 117 | end 118 | 119 | describe "join_bulk_responses" do 120 | subject { described_class.new(base_options) } 121 | 122 | context "when items key is available" do 123 | require "json" 124 | let(:bulk_response) { 125 | LogStash::Json.load ('[{ 126 | "items": [{ 127 | "delete": { 128 | "_index": "website", 129 | "_type": "blog", 130 | "_id": "123", 131 | "_version": 2, 132 | "status": 200, 133 | "found": true 134 | } 135 | }], 136 | "errors": false 137 | }]') 138 | } 139 | it "should be handled properly" do 140 | s = subject.send(:join_bulk_responses, bulk_response) 141 | expect(s["errors"]).to be false 142 | expect(s["items"].size).to be 1 143 | end 144 | end 145 | 146 | context "when items key is not available" do 147 | require "json" 148 | let(:bulk_response) { 149 | JSON.parse ('[{ 150 | "took": 4, 151 | "errors": false 152 | }]') 153 | } 154 | it "should be handled properly" do 155 | s = subject.send(:join_bulk_responses, bulk_response) 156 | expect(s["errors"]).to be false 157 | expect(s["items"].size).to be 0 158 | end 159 | end 160 | end 161 | 162 | describe "#bulk" do 163 | subject { described_class.new(base_options) } 164 | 165 | require "json" 166 | let(:message) { "hey" } 167 | let(:actions) { [ 168 | ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}], 169 | ]} 170 | 171 | context "if a message is over TARGET_BULK_BYTES" do 172 | let(:target_bulk_bytes) { 20 * 1024 * 1024 } 173 | let(:message) { "a" * (target_bulk_bytes + 1) } 174 | 175 | it "should be handled properly" do 176 | allow(subject).to receive(:join_bulk_responses) 177 | expect(subject).to receive(:bulk_send).once do |data| 178 | expect(data.size).to be > target_bulk_bytes 179 | end 180 | s = subject.send(:bulk, actions) 181 | end 182 | end 183 | 184 | context "with two messages" do 185 | let(:message1) { "hey" } 186 | let(:message2) { "you" } 187 | let(:actions) { [ 188 | ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}], 189 | ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}], 190 | ]} 191 | it "executes one bulk_send operation" do 192 | allow(subject).to receive(:join_bulk_responses) 193 | expect(subject).to receive(:bulk_send).once 194 | s = subject.send(:bulk, actions) 195 | end 196 | 197 | context "if one exceeds TARGET_BULK_BYTES" do 198 | let(:target_bulk_bytes) { 20 * 1024 * 1024 } 199 | let(:message1) { "a" * (target_bulk_bytes + 1) } 200 | it "executes two bulk_send operations" do 201 | allow(subject).to receive(:join_bulk_responses) 202 | expect(subject).to receive(:bulk_send).twice 203 | s = subject.send(:bulk, actions) 204 | end 205 | end 206 | end 207 | end 208 | 209 | describe "sniffing" do 210 | let(:client) { LogStash::Outputs::AmazonElasticSearch::HttpClient.new(base_options.merge(client_opts)) } 211 | 212 | context "with sniffing enabled" do 213 | let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } } 214 | 215 | it "should start the sniffer" do 216 | expect(client.pool.sniffing).to be_truthy 217 | end 218 | end 219 | 220 | context "with sniffing disabled" do 221 | let(:client_opts) { {:sniffing => false} } 222 | 223 | it "should not start the sniffer" do 224 | expect(client.pool.sniffing).to be_falsey 225 | end 226 | end 227 | end 228 | end 229 | -------------------------------------------------------------------------------- /spec/unit/outputs/elasticsearch/template_manager_spec.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/devutils/rspec/spec_helper" 6 | require "logstash/outputs/amazon_es/http_client" 7 | require "java" 8 | require "json" 9 | 10 | describe LogStash::Outputs::AmazonElasticSearch::TemplateManager do 11 | 12 | describe ".default_template_path" do 13 | context "amazon_es 1.x" do 14 | it "chooses the 2x template" do 15 | expect(described_class.default_template_path(1)).to match(/elasticsearch-template-es2x.json/) 16 | end 17 | end 18 | context "amazon_es 2.x" do 19 | it "chooses the 2x template" do 20 | expect(described_class.default_template_path(2)).to match(/elasticsearch-template-es2x.json/) 21 | end 22 | end 23 | context "amazon_es 5.x" do 24 | it "chooses the 5x template" do 25 | expect(described_class.default_template_path(5)).to match(/elasticsearch-template-es5x.json/) 26 | end 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /spec/unit/outputs/elasticsearch_spec.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require_relative "../../../spec/es_spec_helper" 6 | require "flores/random" 7 | require "logstash/outputs/amazon_es" 8 | 9 | describe LogStash::Outputs::AmazonElasticSearch do 10 | subject { described_class.new(options) } 11 | let(:options) { { "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 12 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 13 | let(:maximum_seen_major_version) { rand(100) } 14 | 15 | let(:do_register) { true } 16 | 17 | before(:each) do 18 | if do_register 19 | subject.register 20 | 21 | # Rspec mocks can't handle background threads, so... we can't use any 22 | allow(subject.client.pool).to receive(:start_resurrectionist) 23 | allow(subject.client.pool).to receive(:start_sniffer) 24 | allow(subject.client.pool).to receive(:healthcheck!) 25 | allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version) 26 | subject.client.pool.adapter.manticore.respond_with(:body => "{}") 27 | end 28 | end 29 | 30 | after(:each) do 31 | subject.close 32 | end 33 | 34 | 35 | context "with an active instance" do 36 | let(:options) { 37 | { 38 | "index" => "my-index", 39 | "hosts" => ["localhost"], 40 | "path" => "some-path", 41 | "manage_template" => false, 42 | "port" => 9200, 43 | "protocol" => "http", 44 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 45 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" 46 | } 47 | } 48 | 49 | let(:manticore_urls) { subject.client.pool.urls } 50 | let(:manticore_url) { manticore_urls.first } 51 | 52 | describe "getting a document type" do 53 | context "if document_type isn't set" do 54 | let(:options) { super.merge("document_type" => nil)} 55 | context "for 7.x amazon_es clusters" do 56 | let(:maximum_seen_major_version) { 7 } 57 | it "should return '_doc'" do 58 | expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("_doc") 59 | end 60 | end 61 | 62 | context "for 6.x amazon_es clusters" do 63 | let(:maximum_seen_major_version) { 6 } 64 | it "should return 'doc'" do 65 | expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("doc") 66 | end 67 | end 68 | 69 | context "for < 6.0 amazon_es clusters" do 70 | let(:maximum_seen_major_version) { 5 } 71 | it "should get the type from the event" do 72 | expect(subject.send(:get_event_type, LogStash::Event.new("type" => "foo"))).to eql("foo") 73 | end 74 | end 75 | end 76 | 77 | context "with 'document type set'" do 78 | let(:options) { super.merge("document_type" => "bar")} 79 | it "should get the event type from the 'document_type' setting" do 80 | expect(subject.send(:get_event_type, LogStash::Event.new())).to eql("bar") 81 | end 82 | end 83 | 84 | context "with a bad type event field in a < 6.0 es cluster" do 85 | let(:maximum_seen_major_version) { 5 } 86 | let(:type_arg) { ["foo"] } 87 | let(:result) { subject.send(:get_event_type, LogStash::Event.new("type" => type_arg)) } 88 | 89 | before do 90 | allow(subject.instance_variable_get(:@logger)).to receive(:warn) 91 | result 92 | end 93 | 94 | it "should call @logger.warn and return nil" do 95 | expect(subject.instance_variable_get(:@logger)).to have_received(:warn).with(/Bad event type!/, anything).once 96 | end 97 | 98 | it "should set the type to the stringified value" do 99 | expect(result).to eql(type_arg.to_s) 100 | end 101 | end 102 | end 103 | 104 | describe "with auth" do 105 | let(:user) { "myuser" } 106 | let(:password) { ::LogStash::Util::Password.new("mypassword") } 107 | 108 | shared_examples "an authenticated config" do 109 | it "should set the URL auth correctly" do 110 | expect(manticore_url.user).to eq user 111 | end 112 | end 113 | 114 | context "as part of a URL" do 115 | let(:options) { 116 | super.merge("hosts" => ["http://#{user}:#{password.value}@localhost:9200"]) 117 | } 118 | 119 | include_examples("an authenticated config") 120 | end 121 | 122 | context "as a hash option" do 123 | let(:options) { 124 | super.merge!( 125 | "user" => user, 126 | "password" => password 127 | ) 128 | } 129 | 130 | include_examples("an authenticated config") 131 | end 132 | end 133 | 134 | describe "with path" do 135 | it "should properly create a URI with the path" do 136 | expect(subject.path).to eql(options["path"]) 137 | end 138 | 139 | it "should properly set the path on the HTTP client adding slashes" do 140 | expect(manticore_url.path).to eql("/" + options["path"] + "/") 141 | end 142 | 143 | context "with extra slashes" do 144 | let(:path) { "/slashed-path/ "} 145 | let(:options) { super.merge("path" => "/some-path/") } 146 | 147 | it "should properly set the path on the HTTP client without adding slashes" do 148 | expect(manticore_url.path).to eql(options["path"]) 149 | end 150 | end 151 | 152 | context "with a URI based path" do 153 | let(:options) do 154 | o = super() 155 | o.delete("pat˙h") 156 | o["hosts"] = ["http://localhost:9200/mypath/"] 157 | o["port"] = 9200 158 | o["protocol"] = 'http' 159 | o["aws_access_key_id"] = "AAAAAAAAAAAAAAAAAAAA" 160 | o["aws_secret_access_key"] = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" 161 | o 162 | end 163 | let(:client_host_path) { manticore_url.path } 164 | 165 | 166 | context "with a path option but no URL path" do 167 | let(:options) do 168 | o = super() 169 | o["path"] = "/override/" 170 | o["hosts"] = ["http://localhost:9200"] 171 | o["port"] = 9200 172 | o["protocol"] = 'http' 173 | o["aws_access_key_id"] = "AAAAAAAAAAAAAAAAAAAA" 174 | o["aws_secret_access_key"] = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" 175 | o 176 | end 177 | 178 | it "should initialize without error" do 179 | expect { subject }.not_to raise_error 180 | end 181 | 182 | it "should use the option path" do 183 | expect(client_host_path).to eql("/override/") 184 | end 185 | end 186 | 187 | # If you specify the path in two spots that is an error! 188 | context "with a path option and a URL path" do 189 | let(:do_register) { false } # Register will fail 190 | let(:options) do 191 | o = super() 192 | o["path"] = "/override" 193 | o["hosts"] = ["http://localhost:9200/mypath/"] 194 | o 195 | end 196 | 197 | it "should initialize with an error" do 198 | expect { subject.register }.to raise_error(LogStash::ConfigurationError) 199 | end 200 | end 201 | end 202 | end 203 | 204 | describe "without a port specified" do 205 | let(:options) { super.merge('hosts' => 'localhost') } 206 | it "should properly set the default port (9200) on the HTTP client" do 207 | expect(manticore_url.port).to eql(9200) 208 | end 209 | end 210 | 211 | describe "#multi_receive" do 212 | let(:events) { [double("one"), double("two"), double("three")] } 213 | let(:events_tuples) { [double("one t"), double("two t"), double("three t")] } 214 | 215 | before do 216 | allow(subject).to receive(:retrying_submit).with(anything) 217 | events.each_with_index do |e,i| 218 | et = events_tuples[i] 219 | allow(subject).to receive(:event_action_tuple).with(e).and_return(et) 220 | end 221 | subject.multi_receive(events) 222 | end 223 | 224 | end 225 | 226 | context "429 errors" do 227 | let(:event) { ::LogStash::Event.new("foo" => "bar") } 228 | let(:error) do 229 | ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError.new( 230 | 429, double("url").as_null_object, double("request body"), double("response body") 231 | ) 232 | end 233 | let(:logger) { double("logger").as_null_object } 234 | let(:response) { { :errors => [], :items => [] } } 235 | 236 | before(:each) do 237 | 238 | i = 0 239 | bulk_param = [["index", anything, event.to_hash]] 240 | 241 | allow(subject).to receive(:logger).and_return(logger) 242 | 243 | # Fail the first time bulk is called, succeed the next time 244 | allow(subject.client).to receive(:bulk).with(bulk_param) do 245 | i += 1 246 | if i == 1 247 | raise error 248 | end 249 | end.and_return(response) 250 | subject.multi_receive([event]) 251 | end 252 | 253 | it "should retry the 429 till it goes away" do 254 | expect(subject.client).to have_received(:bulk).twice 255 | end 256 | 257 | it "should log a debug message" do 258 | expect(subject.logger).to have_received(:debug).with(/Encountered a retryable error/i, anything) 259 | end 260 | end 261 | end 262 | 263 | context "with timeout set" do 264 | let(:listener) { Flores::Random.tcp_listener } 265 | let(:port) { listener[2] } 266 | let(:options) do 267 | { 268 | "manage_template" => false, 269 | "hosts" => "localhost:#{port}", 270 | "timeout" => 0.1, # fast timeout, 271 | "port" => listener[2], 272 | "protocol" => "http", 273 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 274 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" 275 | } 276 | end 277 | 278 | before do 279 | # Expect a timeout to be logged. 280 | expect(subject.logger).to receive(:error).with(/Attempted to send a bulk request to Elasticsearch/i, anything).at_least(:once) 281 | expect(subject.client).to receive(:bulk).at_least(:twice).and_call_original 282 | end 283 | 284 | it "should fail after the timeout" do 285 | #pending("This is tricky now that we do healthchecks on instantiation") 286 | Thread.new { subject.multi_receive([LogStash::Event.new]) } 287 | 288 | # Allow the timeout to occur 289 | sleep 6 290 | end 291 | end 292 | 293 | describe "the action option" do 294 | context "with a sprintf action" do 295 | let(:options) { {"action" => "%{myactionfield}" , "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 296 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 297 | 298 | let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") } 299 | 300 | it "should interpolate the requested action value when creating an event_action_tuple" do 301 | expect(subject.event_action_tuple(event).first).to eql("update") 302 | end 303 | end 304 | 305 | context "with a sprintf action equals to update" do 306 | let(:options) { {"action" => "%{myactionfield}", "upsert" => '{"message": "some text"}' , 307 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 308 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 309 | 310 | let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") } 311 | 312 | it "should obtain specific action's params from event_action_tuple" do 313 | expect(subject.event_action_tuple(event)[1]).to include(:_upsert) 314 | end 315 | end 316 | 317 | context "with an invalid action" do 318 | let(:options) { {"action" => "SOME Garbaaage", 319 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 320 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 321 | let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call 322 | 323 | it "should raise a configuration error" do 324 | expect { subject.register }.to raise_error(LogStash::ConfigurationError) 325 | end 326 | end 327 | end 328 | 329 | describe "SSL end to end" do 330 | let(:do_register) { false } # skip the register in the global before block, as is called here. 331 | let(:manticore_double) do 332 | double("manticoreX#{self.inspect}") 333 | end 334 | 335 | before(:each) do 336 | response_double = double("manticore response").as_null_object 337 | # Allow healtchecks 338 | allow(manticore_double).to receive(:head).with(any_args).and_return(response_double) 339 | allow(manticore_double).to receive(:get).with(any_args).and_return(response_double) 340 | allow(manticore_double).to receive(:close) 341 | 342 | allow(::Manticore::Client).to receive(:new).and_return(manticore_double) 343 | subject.register 344 | end 345 | 346 | shared_examples("an encrypted client connection") do 347 | it "should enable SSL in manticore" do 348 | expect(subject.client.pool.urls.map(&:scheme).uniq).to eql(['https']) 349 | end 350 | end 351 | 352 | 353 | context "With the 'ssl' option" do 354 | let(:options) { {"ssl" => true, 355 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 356 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}} 357 | 358 | include_examples("an encrypted client connection") 359 | end 360 | 361 | context "With an https host" do 362 | let(:options) { {"hosts" => "https://localhost", 363 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 364 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 365 | include_examples("an encrypted client connection") 366 | end 367 | end 368 | 369 | describe "retry_on_conflict" do 370 | let(:num_retries) { 123 } 371 | let(:event) { LogStash::Event.new("myactionfield" => "update", "message" => "blah") } 372 | let(:options) { { 'retry_on_conflict' => num_retries , 373 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 374 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 375 | 376 | context "with a regular index" do 377 | let(:options) { super.merge("action" => "index") } 378 | 379 | it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do 380 | allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version) 381 | action, params, event_data = subject.event_action_tuple(event) 382 | expect(params).not_to include({:_retry_on_conflict => num_retries}) 383 | end 384 | end 385 | 386 | context "using a plain update" do 387 | let(:options) { super.merge("action" => "update", "retry_on_conflict" => num_retries, "document_id" => 1) } 388 | 389 | it "should set the retry_on_conflict parameter when creating an event_action_tuple" do 390 | action, params, event_data = subject.event_action_tuple(event) 391 | expect(params).to include({:_retry_on_conflict => num_retries}) 392 | end 393 | end 394 | 395 | context "with a sprintf action that resolves to update" do 396 | let(:options) { super.merge("action" => "%{myactionfield}", "retry_on_conflict" => num_retries, "document_id" => 1) } 397 | 398 | it "should set the retry_on_conflict parameter when creating an event_action_tuple" do 399 | action, params, event_data = subject.event_action_tuple(event) 400 | expect(params).to include({:_retry_on_conflict => num_retries}) 401 | expect(action).to eq("update") 402 | end 403 | end 404 | end 405 | 406 | describe "sleep interval calculation" do 407 | let(:retry_max_interval) { 64 } 408 | let(:options) { { "retry_max_interval" => retry_max_interval , 409 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 410 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 411 | 412 | it "should double the given value" do 413 | expect(subject.next_sleep_interval(2)).to eql(4) 414 | expect(subject.next_sleep_interval(32)).to eql(64) 415 | end 416 | 417 | it "should not increase the value past the max retry interval" do 418 | sleep_interval = 2 419 | 100.times do 420 | sleep_interval = subject.next_sleep_interval(sleep_interval) 421 | expect(sleep_interval).to be <= retry_max_interval 422 | end 423 | end 424 | end 425 | 426 | describe "stale connection check" do 427 | let(:validate_after_inactivity) { 123 } 428 | let(:options) { { "validate_after_inactivity" => validate_after_inactivity , 429 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 430 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 431 | let(:do_register) { false } 432 | 433 | before :each do 434 | allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original 435 | end 436 | 437 | after :each do 438 | subject.close 439 | end 440 | 441 | it "should set the correct http client option for 'validate_after_inactivity'" do 442 | subject.register 443 | expect(::Manticore::Client).to have_received(:new) do |options| 444 | expect(options[:check_connection_timeout]).to eq(validate_after_inactivity) 445 | end 446 | end 447 | end 448 | 449 | describe "custom parameters" do 450 | 451 | let(:manticore_urls) { subject.client.pool.urls } 452 | let(:manticore_url) { manticore_urls.first } 453 | 454 | let(:custom_parameters_hash) { { "id" => 1, "name" => "logstash" } } 455 | let(:custom_parameters_query) { custom_parameters_hash.map {|k,v| "#{k}=#{v}" }.join("&") } 456 | 457 | context "using non-url hosts" do 458 | 459 | let(:options) { 460 | { 461 | "index" => "my-index", 462 | "hosts" => ["localhost:9200"], 463 | "path" => "some-path", 464 | "parameters" => custom_parameters_hash, 465 | "port" => 9200, 466 | "protocol" => "http", 467 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 468 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" 469 | } 470 | } 471 | 472 | it "creates a URI with the added parameters" do 473 | expect(subject.parameters).to eql(custom_parameters_hash) 474 | end 475 | 476 | it "sets the query string on the HTTP client" do 477 | expect(manticore_url.query).to eql(custom_parameters_query) 478 | end 479 | end 480 | 481 | context "using url hosts" do 482 | 483 | context "with embedded query parameters" do 484 | let(:options) { 485 | { "hosts" => ["http://localhost:9200/path?#{custom_parameters_query}"] , 486 | "port" => 9200, 487 | "protocol" => "http", 488 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 489 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} 490 | } 491 | 492 | it "sets the query string on the HTTP client" do 493 | expect(manticore_url.query).to eql(custom_parameters_query) 494 | end 495 | end 496 | 497 | context "with explicit query parameters" do 498 | let(:options) { 499 | { 500 | "hosts" => ["http://localhost:9200/path"], 501 | "parameters" => custom_parameters_hash, 502 | "port" => 9200, 503 | "protocol" => "http", 504 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 505 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" 506 | } 507 | } 508 | 509 | it "sets the query string on the HTTP client" do 510 | expect(manticore_url.query).to eql(custom_parameters_query) 511 | end 512 | end 513 | 514 | context "with explicit query parameters and existing url parameters" do 515 | let(:existing_query_string) { "existing=param" } 516 | let(:options) { 517 | { 518 | "hosts" => ["http://localhost:9200/path?#{existing_query_string}"], 519 | "parameters" => custom_parameters_hash, 520 | "port" => 9200, 521 | "protocol" => "http", 522 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 523 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" 524 | } 525 | } 526 | 527 | it "keeps the existing query string" do 528 | expect(manticore_url.query).to include(existing_query_string) 529 | end 530 | 531 | it "includes the new query string" do 532 | expect(manticore_url.query).to include(custom_parameters_query) 533 | end 534 | 535 | it "appends the new query string to the existing one" do 536 | expect(manticore_url.query).to eql("#{existing_query_string}&#{custom_parameters_query}") 537 | end 538 | end 539 | end 540 | end 541 | 542 | context 'handling amazon_es document-level status meant for the DLQ' do 543 | let(:options) { { "manage_template" => false , 544 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 545 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 546 | let(:logger) { subject.instance_variable_get(:@logger) } 547 | 548 | context 'when @dlq_writer is nil' do 549 | before { subject.instance_variable_set '@dlq_writer', nil } 550 | 551 | context 'resorting to previous behaviour of logging the error' do 552 | context 'getting an invalid_index_name_exception' do 553 | it 'should log at ERROR level' do 554 | expect(logger).to receive(:error).with(/Could not index/, hash_including(:status, :action, :response)) 555 | mock_response = { 'index' => { 'error' => { 'type' => 'invalid_index_name_exception' } } } 556 | subject.handle_dlq_status("Could not index event to Elasticsearch.", 557 | [:action, :params, :event], :some_status, mock_response) 558 | end 559 | end 560 | 561 | context 'when getting any other exception' do 562 | it 'should log at WARN level' do 563 | expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response)) 564 | mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } } 565 | subject.handle_dlq_status("Could not index event to Elasticsearch.", 566 | [:action, :params, :event], :some_status, mock_response) 567 | end 568 | end 569 | 570 | context 'when the response does not include [error]' do 571 | it 'should not fail, but just log a warning' do 572 | expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response)) 573 | mock_response = { 'index' => {} } 574 | expect do 575 | subject.handle_dlq_status("Could not index event to Elasticsearch.", 576 | [:action, :params, :event], :some_status, mock_response) 577 | end.to_not raise_error 578 | end 579 | end 580 | end 581 | end 582 | 583 | # DLQ writer always nil, no matter what I try here. So mocking it all the way 584 | context 'when DLQ is enabled' do 585 | let(:dlq_writer) { double('DLQ writer') } 586 | before { subject.instance_variable_set('@dlq_writer', dlq_writer) } 587 | 588 | # Note: This is not quite the desired behaviour. 589 | # We should still log when sending to the DLQ. 590 | # This shall be solved by another issue, however: logstash-output-amazon_es#772 591 | it 'should send the event to the DLQ instead, and not log' do 592 | expect(dlq_writer).to receive(:write).with(:event, /Could not index/) 593 | mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } } 594 | subject.handle_dlq_status("Could not index event to Elasticsearch.", 595 | [:action, :params, :event], :some_status, mock_response) 596 | end 597 | end 598 | end 599 | 600 | describe "custom headers" do 601 | let(:manticore_options) { subject.client.pool.adapter.manticore.instance_variable_get(:@options) } 602 | 603 | context "when set" do 604 | let(:headers) { { "X-Thing" => "Test" } } 605 | let(:options) { { "custom_headers" => headers , 606 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 607 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 608 | it "should use the custom headers in the adapter options" do 609 | expect(manticore_options[:headers]).to eq(headers) 610 | end 611 | end 612 | 613 | context "when not set" do 614 | it "should have no headers" do 615 | expect(manticore_options[:headers]).to be_empty 616 | end 617 | end 618 | end 619 | end 620 | -------------------------------------------------------------------------------- /spec/unit/outputs/error_whitelist_spec.rb: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # See NOTICE for attribution details. 3 | # Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | require "logstash/outputs/amazon_es" 6 | require_relative "../../../spec/es_spec_helper" 7 | 8 | describe "whitelisting error types in expected behavior" do 9 | let(:template) { '{"template" : "not important, will be updated by :index"}' } 10 | let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") } 11 | let(:action1) { ["index", {:_id=>1, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"doc"}, event1] } 12 | let(:settings) { {"manage_template" => true, 13 | "index" => "logstash-2014.11.17", 14 | "template_overwrite" => true, 15 | "hosts" => get_host_port(), 16 | "protocol" => "http", 17 | "port" => 9200, 18 | "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA", 19 | "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} } 20 | 21 | subject { LogStash::Outputs::AmazonElasticSearch.new(settings) } 22 | 23 | before :each do 24 | allow(subject.logger).to receive(:warn) 25 | 26 | subject.register 27 | 28 | allow(subject.client).to receive(:maximum_seen_major_version).and_return(0) 29 | allow(subject.client).to receive(:bulk).and_return( 30 | { 31 | "errors" => true, 32 | "items" => [{ 33 | "create" => { 34 | "status" => 409, 35 | "error" => { 36 | "type" => "document_already_exists_exception", 37 | "reason" => "[shard] document already exists" 38 | } 39 | } 40 | }] 41 | }) 42 | 43 | subject.multi_receive([event1]) 44 | end 45 | 46 | after :each do 47 | subject.close 48 | end 49 | 50 | describe "when failure logging is enabled for everything" do 51 | it "should log a failure on the action" do 52 | expect(subject.logger).to have_received(:warn).with("Failed action.", anything) 53 | end 54 | end 55 | 56 | describe "when failure logging is disabled for docuemnt exists error" do 57 | let(:settings) { super.merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) } 58 | 59 | it "should log a failure on the action" do 60 | expect(subject.logger).not_to have_received(:warn).with("Failed action.", anything) 61 | end 62 | end 63 | 64 | end 65 | --------------------------------------------------------------------------------