├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTORS ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── lib └── logstash │ └── filters │ └── rest.rb ├── logstash-filter-rest.gemspec └── spec └── filters └── rest_spec.rb /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: ruby 3 | cache: bundler 4 | rvm: 5 | - jruby-1.7.25 6 | jdk: 7 | - oraclejdk8 8 | script: 9 | - bundle exec rspec spec 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.5.4 2 | 3 | - update `gemspec` to work with logstash 5.5 4 | 5 | ## 0.5.3 6 | - freeze all instance variables 7 | - fix parallel processing by creating a `deep_clone` for each event 8 | - use `LogStash::Util.deep_clone` for object cloning 9 | - only dump body as json, if json is enabled in config (default) 10 | - delete empty target testcase, as catched by upper logstash `LogStash::ConfigurationError` 11 | - fix `sprintf` find and merge for more complex structures 12 | 13 | ## 0.5.2 14 | - Fix behavior, where a referenced field (`%{...}`) has `ruby` chars 15 | (i.e., consisting of `:`) 16 | - Field interpolation is done by assigning explicit values instead 17 | of converting the `sprintf` string back into a `hash` 18 | 19 | ## 0.5.0 20 | - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 21 | - Require devutils >= 0 to make `bundler` update the package 22 | - Use Event API for LS-5 23 | - Implicit `sprintf`, deprecating the setting 24 | - `target` is now required, dropping support to write into top-level in favor of only using new Event API 25 | - this follows other logstash-plugins like `logstash-filter-json` 26 | - if the response is empty, add the restfailure tags 27 | - Some logging moved before code 28 | - Testcases adapted to new behavior with error check 29 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The following is a list of people who have contributed ideas, code, bug 2 | reports, or in general have helped logstash along its way. 3 | 4 | Contributors: 5 | * Aaron Mildenstein (untergeek) 6 | * Pier-Hugues Pellerin (ph) 7 | 8 | Note: If you've sent us patches, bug reports, or otherwise contributed to 9 | Logstash, and you aren't on the list above and want to be, please let us know 10 | and we'll make sure you're here. Contributions from folks like you are what make 11 | open source awesome. 12 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012–2015 Elasticsearch 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Logstash REST Filter [![Build Status](https://travis-ci.org/gandalfb/logstash-filter-rest.svg?branch=master)](https://travis-ci.org/gandalfb/logstash-filter-rest) 2 | 3 | This is a filter plugin for [Logstash](https://github.com/elasticsearch/logstash). 4 | 5 | It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. 6 | 7 | ## Documentation 8 | 9 | This logstash filter provides an easy way to access RESTful Resources within logstash. It can be used to post data to a REST API or to gather data and save it in your log file. 10 | 11 | ## Usage 12 | ### 1. Installation 13 | You can use the built-in plugin tool of Logstash to install the filter: 14 | ``` 15 | $LS_HOME/bin/logstash-plugin install logstash-filter-rest 16 | ``` 17 | 18 | Or you can build it yourself: 19 | ``` 20 | git clone https://github.com/lucashenning/logstash-filter-rest.git 21 | bundle install 22 | gem build logstash-filter-rest.gemspec 23 | $LS_HOME/bin/logstash-plugin install logstash-filter-rest-0.1.0.gem 24 | ``` 25 | 26 | ### 2. Filter Configuration 27 | Add the following inside the filter section of your logstash configuration: 28 | 29 | ```sh 30 | filter { 31 | rest { 32 | request => { 33 | url => "http://example.com" # string (required, with field reference: "http://example.com?id=%{id}" or params, if defined) 34 | method => "post" # string (optional, default = "get") 35 | headers => { # hash (optional) 36 | "key1" => "value1" 37 | "key2" => "value2" 38 | } 39 | auth => { 40 | user => "AzureDiamond" 41 | password => "hunter2" 42 | } 43 | params => { # hash (optional, available for method => "get" and "post"; if post it will be transformed into body hash and posted as json) 44 | "key1" => "value1" 45 | "key2" => "value2" 46 | "key3" => "%{somefield}" # sprintf is used implicitly 47 | } 48 | } 49 | json => true # boolean (optional, default = true) 50 | target => "my_key" # string (mandatory, no default) 51 | fallback => { # hash describing a default in case of error 52 | "key1" => "value1" 53 | "key2" => "value2" 54 | } 55 | } 56 | } 57 | ``` 58 | 59 | Print plugin version: 60 | 61 | ``` bash 62 | bin/logstash-plugin list --verbose | grep rest 63 | ``` 64 | 65 | Examples for running logstash from `cli`: 66 | 67 | ``` bash 68 | bin/logstash --debug -e 'input { stdin{} } filter { rest { request => { url => "https://jsonplaceholder.typicode.com/posts" method => "post" params => { "userId" => "%{message}" } headers => { "Content-Type" => "application/json" } } target => 'rest' } } output {stdout { codec => rubydebug }}' 69 | ``` 70 | 71 | ``` bash 72 | bin/logstash --debug -e 'input { stdin{} } filter { rest { request => { url => "https://jsonplaceholder.typicode.com/posts" method => "post" body => { "userId" => "%{message}" } headers => { "Content-Type" => "application/json" } } target => 'rest' } } output {stdout { codec => rubydebug }}' 73 | ``` 74 | 75 | ``` bash 76 | bin/logstash --debug -e 'input { stdin{} } filter { rest { request => { url => "http://jsonplaceholder.typicode.com/users/%{message}" } target => 'rest' } } output {stdout { codec => rubydebug }}' 77 | ``` 78 | 79 | ``` bash 80 | bin/logstash --debug -e 'input { stdin{} } filter { rest { request => { url => "https://jsonplaceholder.typicode.com/posts" method => "get" params => { "userId" => "%{message}" } headers => { "Content-Type" => "application/json" } } target => 'rest' } } output {stdout { codec => rubydebug }}' 81 | ``` 82 | 83 | 84 | ## Contributing 85 | 86 | All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin. 87 | 88 | Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here. 89 | 90 | It is more important to the community that you are able to contribute. 91 | 92 | For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file. 93 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "logstash/devutils/rake" 2 | -------------------------------------------------------------------------------- /lib/logstash/filters/rest.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require 'logstash/filters/base' 3 | require 'logstash/namespace' 4 | require 'logstash/plugin_mixins/http_client' 5 | require 'logstash/json' 6 | 7 | # Monkey Patch hsh with a recursive compact and deep freeze 8 | class Hash 9 | def compact 10 | delete_if { |_k, v| v.respond_to?(:each) ? v.compact.empty? : v.nil? } 11 | end 12 | 13 | def deep_freeze 14 | each { |_k, v| v.deep_freeze if v.respond_to? :deep_freeze } 15 | freeze 16 | end 17 | end 18 | 19 | # Monkey Patch Array with deep freeze 20 | class Array 21 | def compact 22 | delete_if { |v| v.respond_to?(:each) ? v.compact.empty? : v.nil? } 23 | end 24 | 25 | def deep_freeze 26 | each { |j| j.deep_freeze if j.respond_to? :deep_freeze } 27 | freeze 28 | end 29 | end 30 | 31 | # Logstash REST Filter 32 | # This filter calls a defined URL and saves the answer into a specified field. 33 | # 34 | class LogStash::Filters::Rest < LogStash::Filters::Base 35 | include LogStash::PluginMixins::HttpClient 36 | 37 | config_name 'rest' 38 | 39 | # Configure the rest request send via HttpClient Plugin 40 | # with hash objects used by the mixin plugin 41 | # 42 | # For example, if you want the data to be put in the `doc` field: 43 | # [source,ruby] 44 | # filter { 45 | # rest { 46 | # request => { 47 | # url => "http://example.com" # string (required, with field reference: "http://example.com?id=%{id}" or params, if defined) 48 | # method => "post" # string (optional, default = "get") 49 | # headers => { # hash (optional) 50 | # "key1" => "value1" 51 | # "key2" => "value2" 52 | # } 53 | # auth => { 54 | # user => "AzureDiamond" 55 | # password => "hunter2" 56 | # } 57 | # params => { # hash (optional, available for method => "get" and "post"; if post it will be transformed into body hash and posted as json) 58 | # "key1" => "value1" 59 | # "key2" => "value2" 60 | # "key3" => "%{somefield}" # Field references are found implicitly on startup 61 | # } 62 | # } 63 | # target => "doc" 64 | # } 65 | # } 66 | # 67 | # NOTE: for further details, please reference https://github.com/logstash-plugins/logstash-mixin-http_client[logstash-mixin-http_client] 68 | config :request, :validate => :hash, :required => true 69 | 70 | # The plugin is written json centric, which defaults to true 71 | # the response body will be parsed to json if true 72 | # 73 | # [source,ruby] 74 | # filter { 75 | # rest { 76 | # json => true 77 | # } 78 | # } 79 | config :json, :validate => :boolean, :default => true 80 | 81 | # If true, references to event fields can be made in 82 | # url, params or body by using '%{somefield}' 83 | # 84 | # [source,ruby] 85 | # filter { 86 | # rest { 87 | # request => { .. } 88 | # sprintf => true 89 | # } 90 | # } 91 | config :sprintf, :validate => :boolean, :default => false, :deprecated => true 92 | 93 | # Define the target field for placing the response data. This setting is 94 | # required and may not be omitted. It is not possible to place the response 95 | # into the event top-level. 96 | # 97 | # For example, if you want the data to be put in the `doc` field: 98 | # [source,ruby] 99 | # filter { 100 | # rest { 101 | # target => "doc" 102 | # } 103 | # } 104 | # 105 | # Rest response will be expanded into a data structure in the `target` field. 106 | # 107 | # NOTE: if the `target` field already exists, it will be overwritten! 108 | config :target, :validate => :string, :required => true 109 | 110 | # If set, any error like json parsing or invalid http response 111 | # will result in this hash to be added to target instead of error tags 112 | # 113 | # For example, if you want the fallback data to be put in the `target` field: 114 | # [source,ruby] 115 | # filter { 116 | # rest { 117 | # fallback => { 118 | # 'key1' => 'value1' 119 | # 'key2' => 'value2' 120 | # ... 121 | # } 122 | # } 123 | # } 124 | config :fallback, :validate => :hash, :default => {} 125 | 126 | # Append values to the `tags` field when there has been no 127 | # successful match or json parsing error 128 | config :tag_on_rest_failure, :validate => :array, :default => ['_restfailure'] 129 | config :tag_on_json_failure, :validate => :array, :default => ['_jsonparsefailure'] 130 | 131 | public 132 | 133 | def register 134 | @request = normalize_request(@request).deep_freeze 135 | @sprintf_fields = find_sprintf( 136 | LogStash::Util.deep_clone(@request) 137 | ).deep_freeze 138 | @sprintf_needed = !@sprintf_fields.empty? 139 | @target = normalize_target(@target).freeze 140 | end # def register 141 | 142 | private 143 | 144 | def normalize_target(target) 145 | # make sure @target is in the format [field name] if defined, 146 | # i.e. not empty and surrounded by brakets 147 | raise LogStash::ConfigurationError, 'target config string is empty, please set a valid field name' if target.empty? 148 | target = "[#{target}]" if target && target !~ /^\[[^\[\]]+\]$/ 149 | target 150 | end 151 | 152 | private 153 | 154 | def normalize_request(url_or_spec) 155 | if url_or_spec.is_a?(String) 156 | res = [:get, url_or_spec] 157 | elsif url_or_spec.is_a?(Hash) 158 | # The client will expect keys / values 159 | spec = Hash[url_or_spec.clone.map { |k, v| [k.to_sym, v] }] 160 | 161 | # method and url aren't really part of the options, so we pull them out 162 | method = (spec.delete(:method) || :get).to_sym.downcase 163 | url = spec.delete(:url) 164 | 165 | # if it is a post and json, it is used as body string, not params 166 | spec[:body] = spec.delete(:params) if method == :post && spec[:params] 167 | 168 | # We need these strings to be keywords! 169 | spec[:auth] = { user: spec[:auth]['user'], pass: spec[:auth]['password'] } if spec[:auth] 170 | 171 | res = [method.freeze, url, spec] 172 | else 173 | raise LogStash::ConfigurationError, "Invalid URL or request spec: '#{url_or_spec}', expected a String or Hash!" 174 | end 175 | 176 | validate_request!(url_or_spec, res) 177 | res 178 | end 179 | 180 | private 181 | 182 | def validate_request!(url_or_spec, request) 183 | method, url, spec = request 184 | 185 | raise LogStash::ConfigurationError, "No URL provided for request! #{url_or_spec}" unless url 186 | raise LogStash::ConfigurationError, "Not supported request method #{method}" unless [ :get, :post ].include?( method ) 187 | 188 | if spec && spec[:auth] 189 | raise LogStash::ConfigurationError, "Auth was specified, but 'user' was not!" unless spec[:auth][:user] 190 | raise LogStash::ConfigurationError, "Auth was specified, but 'password' was not!" unless spec[:auth][:pass] 191 | end 192 | 193 | request 194 | end 195 | 196 | private 197 | 198 | def find_sprintf(config) 199 | field_matcher = /%\{[^}]+\}/ 200 | if config.is_a?(Hash) 201 | config.keep_if do |_k, v| 202 | find_sprintf(v) 203 | end.compact 204 | elsif config.is_a?(Array) 205 | config.keep_if do |v| 206 | find_sprintf(v) 207 | end.compact 208 | elsif config.is_a?(String) && config =~ field_matcher 209 | config 210 | end 211 | end 212 | 213 | private 214 | 215 | def request_http(request) 216 | if request[2].key?(:body) && @json 217 | request[2][:body] = LogStash::Json.dump(request[2][:body]) 218 | end 219 | @logger.debug? && @logger.debug('fetching request', 220 | :request => request) 221 | 222 | method, url, *request_opts = request 223 | response = client.http(method, url, *request_opts) 224 | [response.code, response.body] 225 | end 226 | 227 | private 228 | 229 | def process_response(response, event) 230 | if @json 231 | begin 232 | parsed = LogStash::Json.load(response) 233 | if parsed.empty? 234 | @logger.warn('rest response empty', 235 | :response => response, :event => event) 236 | @tag_on_rest_failure.each { |tag| event.tag(tag) } 237 | else 238 | event.set(@target, parsed) 239 | end 240 | rescue 241 | if @fallback.empty? 242 | @logger.warn('JSON parsing error', 243 | :response => response, :event => event) 244 | @tag_on_json_failure.each { |tag| event.tag(tag) } 245 | else 246 | event.set(@target, @fallback) 247 | end 248 | end 249 | else 250 | event.set(@target, response.strip) 251 | end 252 | end 253 | 254 | private 255 | 256 | def field_intrpl(intrpl_fields, event) 257 | case intrpl_fields 258 | when String 259 | result = event.sprintf(intrpl_fields) 260 | when Array 261 | result = [] 262 | intrpl_fields.each do |v| 263 | result << field_intrpl(v, event) 264 | end 265 | when Hash 266 | result = {} 267 | intrpl_fields.each do |k, v| 268 | result[k] = field_intrpl(v, event) 269 | end 270 | else 271 | result = intrpl_fields 272 | end 273 | result 274 | end 275 | 276 | public 277 | 278 | def filter(event) 279 | return unless filter?(event) 280 | request = LogStash::Util.deep_clone(@request) 281 | @logger.debug? && @logger.debug('processing request', 282 | :request => request, 283 | :sprintf_needed => @sprintf_needed) 284 | 285 | if @sprintf_needed 286 | request = field_intrpl(request, event) 287 | @logger.debug? && @logger.debug('interpolated request', 288 | :request => request) 289 | end 290 | 291 | client_error = nil 292 | begin 293 | code, body = request_http(request) 294 | rescue StandardError => e 295 | client_error = e 296 | end 297 | 298 | if !client_error && code.between?(200, 299) 299 | @logger.debug? && @logger.debug('success received', 300 | :code => code, :body => body) 301 | process_response(body, event) 302 | else 303 | @logger.debug? && @logger.debug('http error received', 304 | :code => code, :body => body, 305 | :client_error => client_error) 306 | if @fallback.empty? 307 | @logger.error('error in rest filter', 308 | :request => request, :json => @json, 309 | :code => code, :body => body, 310 | :client_error => client_error) 311 | @tag_on_rest_failure.each { |tag| event.tag(tag) } 312 | else 313 | @logger.debug? && @logger.debug('setting fallback', 314 | :fallback => @fallback) 315 | event.set(@target, @fallback) 316 | end 317 | end 318 | filter_matched(event) 319 | end # def filter 320 | end # class LogStash::Filters::Rest 321 | -------------------------------------------------------------------------------- /logstash-filter-rest.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = 'logstash-filter-rest' 3 | s.version = '0.5.5' 4 | s.licenses = ['Apache License (2.0)'] 5 | s.summary = 'This filter requests data from a RESTful Web Service.' 6 | s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-rest. This gem is not a stand-alone program' 7 | s.authors = ['Lucas Henning', 'Gandalf Buscher'] 8 | s.email = 'mail@hurb.de' 9 | s.homepage = 'https://github.com/lucashenning/logstash-filter-rest/' 10 | s.require_paths = ['lib'] 11 | 12 | # Files 13 | s.files = Dir['lib/**/*', 14 | 'spec/**/*', 15 | 'vendor/**/*', 16 | '*.gemspec', 17 | '*.md', 18 | 'CONTRIBUTORS', 19 | 'Gemfile', 20 | 'LICENSE', 21 | 'NOTICE.TXT'] 22 | # Tests 23 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 24 | 25 | # Special flag to let us know this is actually a logstash plugin 26 | s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'filter' } 27 | 28 | # Gem dependencies 29 | s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99' 30 | s.add_runtime_dependency 'logstash-mixin-http_client', '>= 5.0.0', '< 9.0.0' 31 | 32 | s.add_development_dependency 'logstash-devutils', '>= 0', '< 2.0.0' 33 | end 34 | -------------------------------------------------------------------------------- /spec/filters/rest_spec.rb: -------------------------------------------------------------------------------- 1 | require 'logstash/devutils/rspec/spec_helper' 2 | require 'logstash/filters/rest' 3 | 4 | describe LogStash::Filters::Rest do 5 | describe 'Set to Rest Filter Get without params' do 6 | let(:config) do <<-CONFIG 7 | filter { 8 | rest { 9 | request => { 10 | url => 'http://jsonplaceholder.typicode.com/users/10' 11 | } 12 | json => true 13 | target => 'rest' 14 | } 15 | } 16 | CONFIG 17 | end 18 | 19 | sample('message' => 'some text') do 20 | expect(subject).to include('rest') 21 | expect(subject.get('rest')).to include('id') 22 | expect(subject.get('[rest][id]')).to eq(10) 23 | expect(subject.get('rest')).to_not include('fallback') 24 | end 25 | end 26 | describe 'Set to Rest Filter Get without params custom target' do 27 | let(:config) do <<-CONFIG 28 | filter { 29 | rest { 30 | request => { 31 | url => 'http://jsonplaceholder.typicode.com/users/10' 32 | } 33 | json => true 34 | target => 'testing' 35 | } 36 | } 37 | CONFIG 38 | end 39 | 40 | sample('message' => 'some text') do 41 | expect(subject).to include('testing') 42 | expect(subject.get('testing')).to include('id') 43 | expect(subject.get('[testing][id]')).to eq(10) 44 | expect(subject.get('testing')).to_not include('fallback') 45 | end 46 | end 47 | describe 'Set to Rest Filter Get without params and sprintf' do 48 | let(:config) do <<-CONFIG 49 | filter { 50 | rest { 51 | request => { 52 | url => "http://jsonplaceholder.typicode.com/users/%{message}" 53 | } 54 | json => true 55 | sprintf => true 56 | target => 'rest' 57 | } 58 | } 59 | CONFIG 60 | end 61 | 62 | sample('message' => '10') do 63 | expect(subject).to include('rest') 64 | expect(subject.get('rest')).to include('id') 65 | expect(subject.get('[rest][id]')).to eq(10) 66 | expect(subject.get('rest')).to_not include('fallback') 67 | end 68 | sample('message' => '9') do 69 | expect(subject).to include('rest') 70 | expect(subject.get('rest')).to include('id') 71 | expect(subject.get('[rest][id]')).to eq(9) 72 | expect(subject.get('rest')).to_not include('fallback') 73 | end 74 | end 75 | describe 'Set to Rest Filter Get without params http error' do 76 | let(:config) do <<-CONFIG 77 | filter { 78 | rest { 79 | request => { 80 | url => 'http://httpstat.us/404' 81 | } 82 | json => true 83 | target => 'rest' 84 | } 85 | } 86 | CONFIG 87 | end 88 | 89 | sample('message' => 'some text') do 90 | expect(subject).to_not include('rest') 91 | expect(subject.get('tags')).to include('_restfailure') 92 | end 93 | end 94 | describe 'Set to Rest Filter Get with params' do 95 | let(:config) do <<-CONFIG 96 | filter { 97 | rest { 98 | request => { 99 | url => 'https://jsonplaceholder.typicode.com/posts' 100 | params => { 101 | userId => 10 102 | } 103 | headers => { 104 | 'Content-Type' => 'application/json' 105 | } 106 | } 107 | json => true 108 | target => 'rest' 109 | } 110 | } 111 | CONFIG 112 | end 113 | 114 | sample('message' => 'some text') do 115 | expect(subject).to include('rest') 116 | expect(subject.get('[rest][0]')).to include('userId') 117 | expect(subject.get('[rest][0][userId]')).to eq(10) 118 | expect(subject.get('rest')).to_not include('fallback') 119 | end 120 | end 121 | describe 'empty response' do 122 | let(:config) do <<-CONFIG 123 | filter { 124 | rest { 125 | request => { 126 | url => 'https://jsonplaceholder.typicode.com/posts' 127 | params => { 128 | userId => 0 129 | } 130 | headers => { 131 | 'Content-Type' => 'application/json' 132 | } 133 | } 134 | target => 'rest' 135 | } 136 | } 137 | CONFIG 138 | end 139 | 140 | sample('message' => 'some text') do 141 | expect(subject).to_not include('rest') 142 | expect(subject.get('tags')).to include('_restfailure') 143 | end 144 | end 145 | describe 'Set to Rest Filter Get with params sprintf' do 146 | let(:config) do <<-CONFIG 147 | filter { 148 | rest { 149 | request => { 150 | url => 'https://jsonplaceholder.typicode.com/posts' 151 | params => { 152 | userId => "%{message}" 153 | id => "%{message}" 154 | } 155 | headers => { 156 | 'Content-Type' => 'application/json' 157 | } 158 | } 159 | json => true 160 | target => 'rest' 161 | } 162 | } 163 | CONFIG 164 | end 165 | 166 | sample('message' => '1') do 167 | expect(subject).to include('rest') 168 | expect(subject.get('[rest][0]')).to include('userId') 169 | expect(subject.get('[rest][0][userId]')).to eq(1) 170 | expect(subject.get('[rest][0][id]')).to eq(1) 171 | expect(subject.get('rest').length).to eq(1) 172 | expect(subject.get('rest')).to_not include('fallback') 173 | end 174 | end 175 | describe 'Set to Rest Filter Post with params' do 176 | let(:config) do <<-CONFIG 177 | filter { 178 | rest { 179 | request => { 180 | url => 'https://jsonplaceholder.typicode.com/posts' 181 | method => 'post' 182 | params => { 183 | title => 'foo' 184 | body => 'bar' 185 | userId => 42 186 | } 187 | headers => { 188 | 'Content-Type' => 'application/json' 189 | } 190 | } 191 | json => true 192 | target => 'rest' 193 | } 194 | } 195 | CONFIG 196 | end 197 | 198 | sample('message' => 'some text') do 199 | expect(subject).to include('rest') 200 | expect(subject.get('rest')).to include('id') 201 | expect(subject.get('[rest][userId]')).to eq(42) 202 | expect(subject.get('rest')).to_not include('fallback') 203 | end 204 | end 205 | describe 'Set to Rest Filter Post with params sprintf' do 206 | let(:config) do <<-CONFIG 207 | filter { 208 | rest { 209 | request => { 210 | url => 'https://jsonplaceholder.typicode.com/posts' 211 | method => 'post' 212 | params => { 213 | title => '%{message}' 214 | body => 'bar' 215 | userId => "%{message}" 216 | } 217 | headers => { 218 | 'Content-Type' => 'application/json' 219 | } 220 | } 221 | json => true 222 | target => 'rest' 223 | } 224 | } 225 | CONFIG 226 | end 227 | 228 | sample('message' => '42') do 229 | expect(subject).to include('rest') 230 | expect(subject.get('rest')).to include('id') 231 | expect(subject.get('[rest][title]')).to eq(42) 232 | expect(subject.get('[rest][userId]')).to eq(42) 233 | expect(subject.get('rest')).to_not include('fallback') 234 | end 235 | sample('message' => ':5e?#!-_') do 236 | expect(subject).to include('rest') 237 | expect(subject.get('rest')).to include('id') 238 | expect(subject.get('[rest][title]')).to eq(':5e?#!-_') 239 | expect(subject.get('[rest][userId]')).to eq(':5e?#!-_') 240 | expect(subject.get('rest')).to_not include('fallback') 241 | end 242 | sample('message' => ':4c43=>') do 243 | expect(subject).to include('rest') 244 | expect(subject.get('rest')).to include('id') 245 | expect(subject.get('[rest][title]')).to eq(':4c43=>') 246 | expect(subject.get('[rest][userId]')).to eq(':4c43=>') 247 | expect(subject.get('rest')).to_not include('fallback') 248 | end 249 | end 250 | describe 'Set to Rest Filter Post with body sprintf' do 251 | let(:config) do <<-CONFIG 252 | filter { 253 | rest { 254 | request => { 255 | url => 'https://jsonplaceholder.typicode.com/posts' 256 | method => 'post' 257 | body => { 258 | title => 'foo' 259 | body => 'bar' 260 | userId => "%{message}" 261 | } 262 | headers => { 263 | 'Content-Type' => 'application/json' 264 | } 265 | } 266 | json => true 267 | target => 'rest' 268 | } 269 | } 270 | CONFIG 271 | end 272 | 273 | sample('message' => '42') do 274 | expect(subject).to include('rest') 275 | expect(subject.get('rest')).to include('id') 276 | expect(subject.get('[rest][userId]')).to eq(42) 277 | expect(subject.get('rest')).to_not include('fallback') 278 | end 279 | end 280 | describe 'Set to Rest Filter Post with body sprintf nested params' do 281 | let(:config) do <<-CONFIG 282 | filter { 283 | rest { 284 | request => { 285 | url => 'https://jsonplaceholder.typicode.com/posts' 286 | method => 'post' 287 | body => { 288 | key1 => [ 289 | { 290 | "filterType" => "text" 291 | "text" => "salmon" 292 | "boolean" => false 293 | }, 294 | { 295 | "filterType" => "unique" 296 | } 297 | ] 298 | key2 => [ 299 | { 300 | "message" => "123%{message}" 301 | "boolean" => true 302 | } 303 | ] 304 | key3 => [ 305 | { 306 | "text" => "%{message}123" 307 | "filterType" => "text" 308 | "number" => 44 309 | }, 310 | { 311 | "filterType" => "unique" 312 | "null" => nil 313 | } 314 | ] 315 | userId => "%{message}" 316 | } 317 | headers => { 318 | 'Content-Type' => 'application/json' 319 | } 320 | } 321 | target => 'rest' 322 | } 323 | } 324 | CONFIG 325 | end 326 | 327 | sample('message' => '42') do 328 | expect(subject).to include('rest') 329 | expect(subject.get('rest')).to include('key1') 330 | expect(subject.get('[rest][key1][0][boolean]')).to eq('false') 331 | expect(subject.get('[rest][key1][1][filterType]')).to eq('unique') 332 | expect(subject.get('[rest][key2][0][message]')).to eq('12342') 333 | expect(subject.get('[rest][key2][0][boolean]')).to eq('true') 334 | expect(subject.get('[rest][key3][0][text]')).to eq('42123') 335 | expect(subject.get('[rest][key3][0][filterType]')).to eq('text') 336 | expect(subject.get('[rest][key3][0][number]')).to eq(44) 337 | expect(subject.get('[rest][key3][1][filterType]')).to eq('unique') 338 | expect(subject.get('[rest][key3][1][null]')).to eq('nil') 339 | expect(subject.get('[rest][userId]')).to eq(42) 340 | expect(subject.get('rest')).to_not include('fallback') 341 | end 342 | end 343 | describe 'fallback' do 344 | let(:config) do <<-CONFIG 345 | filter { 346 | rest { 347 | request => { 348 | url => 'http://jsonplaceholder.typicode.com/users/0' 349 | } 350 | json => true 351 | fallback => { 352 | 'fallback1' => true 353 | 'fallback2' => true 354 | } 355 | target => 'rest' 356 | } 357 | } 358 | CONFIG 359 | end 360 | 361 | sample('message' => 'some text') do 362 | expect(subject).to include('rest') 363 | expect(subject.get('rest')).to include('fallback1') 364 | expect(subject.get('rest')).to include('fallback2') 365 | expect(subject.get('rest')).to_not include('id') 366 | end 367 | end 368 | describe 'empty target exception' do 369 | let(:config) do <<-CONFIG 370 | filter { 371 | rest { 372 | request => { 373 | url => 'http://jsonplaceholder.typicode.com/users/0' 374 | } 375 | json => true 376 | fallback => { 377 | 'fallback1' => true 378 | 'fallback2' => true 379 | } 380 | target => '' 381 | } 382 | } 383 | CONFIG 384 | end 385 | sample('message' => 'some text') do 386 | expect { subject }.to raise_error(LogStash::ConfigurationError) 387 | end 388 | end 389 | describe 'http client throws exception' do 390 | let(:config) do <<-CONFIG 391 | filter { 392 | rest { 393 | request => { 394 | url => 'invalid_url' 395 | } 396 | target => 'rest' 397 | } 398 | } 399 | CONFIG 400 | end 401 | sample('message' => 'some text') do 402 | expect(subject).to_not include('rest') 403 | expect(subject.get('tags')).to include('_restfailure') 404 | end 405 | end 406 | end 407 | --------------------------------------------------------------------------------