├── .github └── workflows │ └── test.yml ├── .gitignore ├── CHANGELOG.md ├── Gemfile ├── ISSUE_TEMPLATE.md ├── LICENSE.txt ├── README.md ├── Rakefile ├── fluent-plugin-out-http.gemspec ├── lib └── fluent │ ├── plugin │ └── out_http.rb │ └── test │ └── http_output_test.rb ├── run_test.rb └── test └── plugin ├── script └── plugin │ └── formatter_test.rb └── test_out_http.rb /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | - push 4 | - pull_request 5 | jobs: 6 | build: 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | ruby: [ '2.7', '3.0', '3.1' ] 12 | os: 13 | - ubuntu-latest 14 | - macOS-latest 15 | name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} 16 | steps: 17 | - uses: actions/checkout@v3 18 | - uses: ruby/setup-ruby@v1 19 | with: 20 | ruby-version: ${{ matrix.ruby }} 21 | bundler-cache: true 22 | - name: unit testing 23 | run: | 24 | bundle exec rake test 25 | 26 | gem: 27 | name: Gem test 28 | runs-on: ubuntu-latest 29 | steps: 30 | - uses: actions/checkout@v3 31 | - uses: ruby/setup-ruby@v1 32 | with: 33 | ruby-version: 3.1 34 | - name: Install 35 | run: | 36 | rake install 37 | gem install test-unit webrick 38 | - name: Test 39 | run: | 40 | mkdir -p tmp 41 | cd tmp 42 | cp -a ../test/ ./ 43 | ../run_test.rb 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | \#* 3 | .\#* 4 | *.gem 5 | .bundle 6 | .ruby-version 7 | Gemfile.lock 8 | vendor 9 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | ## 1.3.4 3 | * Add debug log about chunk writing 4 | * Tweak CI 5 | 6 | ## 1.3.3 7 | * Revert x-ndjson format payload behavior 8 | 9 | ## 1.3.2 10 | * Fix invalid x-ndjson payload format 11 | 12 | ## 1.3.1 13 | * Support compression request 14 | 15 | ## 1.3.0 16 | * Support all private key types 17 | * Recoverable error codes 18 | * Bulk request with x-ndjson 19 | 20 | ## 1.2.0 21 | * Support mutual authentication 22 | 23 | ## 1.1.7 24 | * Fix dependent Fluentd version 25 | 26 | ## 1.1.6 27 | * Pass chunk directly info built-in placeholder instead of chunk.metadata 28 | 29 | ## 1.1.5 30 | * Add :raw serializer 31 | 32 | ## 1.1.4 33 | * Add custom formatter feature 34 | * Tweak Travis CI tasks 35 | 36 | ## 1.1.3 37 | * Send query_string to endpoint_url 38 | 39 | ## 1.1.2 40 | * Added custom headers feature 41 | 42 | ## 1.1.1 43 | * Added plain text transport capability 44 | * Added specify cacert file for ssl verify 45 | 46 | ## 1.1.0 47 | * Support for jwt token authentication 48 | 49 | ## 1.0.1 50 | * Added endpoint_url placeholder support 51 | 52 | ## 1.0.0 53 | * Use Fluentd v1 API 54 | 55 | ## 0.2.0 56 | ### Added 57 | * SSL is now supported if `endpoint_url` uses the `https` scheme (uses ruby-2.1 syntax internally) 58 | * New config: set `ssl_no_verify` to `true` to bypass SSL certificate verification. 59 | Use at your own risk. 60 | ### Changed 61 | * Fixed tests: 62 | * Removed some warnings 63 | * Fixed failing binary test to use UTF-8 64 | ### Removed 65 | * Dropped support of Ruby 1.9-2.0 66 | 67 | ## 0.1.4 68 | * #11 Updated Fluentd dependency to: [">= 0.10.0", "< 2"] 69 | * #10 `password` is now marked as a [secret option](https://github.com/fluent/fluentd/pull/604) 70 | 71 | ## 0.1.3 72 | * Added a new configuration option: `raise_on_error` (default: true) 73 | * In order to let the plugin raise exceptions like it did in 0.1.1: keep using your configuration as-is 74 | * In order to suppress all exceptions: add `raise_on_error false` to your configuration 75 | 76 | ## 0.1.2 77 | * #6 Catch all `StandardError`s during HTTP request to prevent td-agent from freezing 78 | 79 | ## 0.1.1 80 | * #2 Use yajl instead of json as json serializer 81 | * #1 Fix a bug where a nil HTTP response caused the plugin to stop working 82 | 83 | ## 0.1.0 84 | * Initial release 85 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in fluent-plugin-out-http.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | #### Problem 2 | 3 | ... 4 | 5 | #### Steps to replicate 6 | 7 | Provide example config and message 8 | 9 | #### Expected Behavior or What you need to ask 10 | 11 | ... 12 | 13 | #### Using Fluentd and out_http plugin versions 14 | 15 | * OS version 16 | * Fluentd v0.12 or v0.14/v1.0 17 | * paste result of ``fluentd --version`` or ``td-agent --version`` 18 | * out_http plugin 1.x.y or 0.x.y 19 | * paste boot log of fluentd or td-agent 20 | * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock 21 | * Bear Metal or Within Docker or Kubernetes or others? (optional) 22 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); 2 | you may not use this file except in compliance with the License. 3 | You may obtain a copy of the License at 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fluent-plugin-out-http, a plugin for [Fluentd](http://fluentd.org) 2 | 3 | A generic [fluentd][1] output plugin for sending logs to an HTTP endpoint. 4 | 5 | **NOTE:** This plugin name is overlapped from Fluentd core plugin which name is `out_http`. But core plugin does not provide buffered/non-buffered switch and batch/non-batch request switch. If you want to use these switch features. Please consider to use this plugin. 6 | 7 | [![Build Status](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-out-http.svg?branch=master)](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-out-http) 8 | 9 | ## Installation 10 | 11 | ### For bundler 12 | 13 | Write the following line in Gemfile: 14 | 15 | ```gemfile 16 | gem "fluent-plugin-out-http" 17 | ``` 18 | 19 | and then, 20 | 21 | ```console 22 | $ bundle install 23 | ``` 24 | 25 | ### for fluentd 26 | 27 | ```console 28 | $ fluent-gem install fluent-plugin-out-http 29 | ``` 30 | 31 | ### For td-agent 32 | 33 | ```console 34 | $ td-agent-gem install fluent-plugin-out-http 35 | ``` 36 | 37 | ## Configuration options 38 | 39 | 40 | @type http 41 | endpoint_url http://localhost.local/api/ 42 | ssl_no_verify false # default: false 43 | http_method put # default: post 44 | serializer json # default: form 45 | rate_limit_msec 100 # default: 0 = no rate limiting 46 | raise_on_error false # default: true 47 | recoverable_status_codes 503, 400 # default: 503 48 | cacert_file /etc/ssl/endpoint1.cert # default: '' 49 | client_cert_path /path/to/client_cert.crt # default: '' 50 | private_key_path /path/to/private_key.key # default: '' 51 | private_key_passphrase yourpassphrase # default: '' 52 | custom_headers {"token":"arbitrary"} # default: nil 53 | authentication basic # default: none 54 | username alice # default: '' 55 | password bobpop # default: '', secret: true 56 | token tokent # default: '' 57 | buffered true # default: false. Switch non-buffered/buffered mode 58 | bulk_request false # default: false. Send events as application/x-ndjson 59 | compress_request true # default: false. Send compressed events 60 | 61 | 62 | ## Usage notes 63 | 64 | If you'd like to retry failed requests, consider using [fluent-plugin-bufferize][3]. 65 | Or, specify appropriate `recoverable_status_codes` parameter. 66 | 67 | To send events with bulk_request, you should specify `bulk_request` as `true` 68 | Note that when this parameter as `true`, Fluentd always send events as `application/x-ndjson`. 69 | Currently, `application/x-ndjson` is only supported MIME type for bulk_request. 70 | 71 | ---- 72 | 73 | Heavily based on [fluent-plugin-growthforecast][2] 74 | 75 | [1]: http://fluentd.org/ 76 | [2]: https://github.com/tagomoris/fluent-plugin-growthforecast 77 | [3]: https://github.com/sabottenda/fluent-plugin-bufferize 78 | 79 | ## How to release 80 | 81 | 1. Update `gem.version` of the `gemspec` file. 82 | 1. Update `CHANGELOG.md`. 83 | 1. Commit them. 84 | 1. `$ bundle exec rake release` 85 | * It will add and push the tag, and publish the gem. 86 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | require "bundler/gem_tasks" 3 | 4 | require 'rake/testtask' 5 | Rake::TestTask.new(:test) do |test| 6 | test.libs << 'lib' << 'test' 7 | test.pattern = 'test/**/test_*.rb' 8 | test.verbose = true 9 | end 10 | 11 | task :default => :test 12 | -------------------------------------------------------------------------------- /fluent-plugin-out-http.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | Gem::Specification.new do |gem| 4 | gem.name = "fluent-plugin-out-http" 5 | gem.version = "1.3.4" 6 | gem.authors = ["Marica Odagaki"] 7 | gem.email = ["ento.entotto@gmail.com"] 8 | gem.summary = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint} 9 | gem.description = gem.summary 10 | gem.homepage = "https://github.com/fluent-plugins-nursery/fluent-plugin-out-http" 11 | gem.licenses = ["Apache-2.0"] 12 | 13 | gem.files = `git ls-files`.split($\) 14 | gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } 15 | gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) 16 | gem.require_paths = ["lib"] 17 | 18 | gem.required_ruby_version = '>= 2.1.0' 19 | 20 | gem.add_runtime_dependency "yajl-ruby", "~> 1.0" 21 | gem.add_runtime_dependency "fluentd", [">= 0.14.22", "< 2"] 22 | gem.add_development_dependency "bundler" 23 | gem.add_development_dependency "rake" 24 | gem.add_development_dependency "test-unit", ">= 3.1.0" 25 | gem.add_development_dependency "webrick" 26 | end 27 | -------------------------------------------------------------------------------- /lib/fluent/plugin/out_http.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'uri' 3 | require 'yajl' 4 | require 'fluent/plugin/output' 5 | require 'tempfile' 6 | require 'openssl' 7 | require 'zlib' 8 | 9 | class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output 10 | Fluent::Plugin.register_output('http', self) 11 | 12 | class RecoverableResponse < StandardError; end 13 | 14 | helpers :compat_parameters, :formatter 15 | 16 | DEFAULT_BUFFER_TYPE = "memory" 17 | DEFAULT_FORMATTER = "json" 18 | 19 | def initialize 20 | super 21 | end 22 | 23 | # Endpoint URL ex. http://localhost.local/api/ 24 | config_param :endpoint_url, :string 25 | 26 | # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE` 27 | config_param :ssl_no_verify, :bool, :default => false 28 | 29 | # HTTP method 30 | config_param :http_method, :enum, list: [:get, :put, :post, :delete], :default => :post 31 | 32 | # form | json | text | raw 33 | config_param :serializer, :enum, list: [:json, :form, :text, :raw], :default => :form 34 | 35 | # Simple rate limiting: ignore any records within `rate_limit_msec` 36 | # since the last one. 37 | config_param :rate_limit_msec, :integer, :default => 0 38 | 39 | # Raise errors that were rescued during HTTP requests? 40 | config_param :raise_on_error, :bool, :default => true 41 | 42 | # Specify recoverable error codes 43 | config_param :recoverable_status_codes, :array, value_type: :integer, default: [503] 44 | 45 | # ca file to use for https request 46 | config_param :cacert_file, :string, :default => '' 47 | 48 | # specify client sertificate 49 | config_param :client_cert_path, :string, :default => '' 50 | 51 | # specify private key path 52 | config_param :private_key_path, :string, :default => '' 53 | 54 | # specify private key passphrase 55 | config_param :private_key_passphrase, :string, :default => '', :secret => true 56 | 57 | # custom headers 58 | config_param :custom_headers, :hash, :default => nil 59 | 60 | # 'none' | 'basic' | 'jwt' | 'bearer' 61 | config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer], :default => :none 62 | config_param :username, :string, :default => '' 63 | config_param :password, :string, :default => '', :secret => true 64 | config_param :token, :string, :default => '' 65 | # Switch non-buffered/buffered plugin 66 | config_param :buffered, :bool, :default => false 67 | config_param :bulk_request, :bool, :default => false 68 | # Compress with gzip except for form serializer 69 | config_param :compress_request, :bool, :default => false 70 | 71 | config_section :buffer do 72 | config_set_default :@type, DEFAULT_BUFFER_TYPE 73 | config_set_default :chunk_keys, ['tag'] 74 | end 75 | 76 | config_section :format do 77 | config_set_default :@type, DEFAULT_FORMATTER 78 | end 79 | 80 | def configure(conf) 81 | compat_parameters_convert(conf, :buffer, :formatter) 82 | super 83 | 84 | @ssl_verify_mode = if @ssl_no_verify 85 | OpenSSL::SSL::VERIFY_NONE 86 | else 87 | OpenSSL::SSL::VERIFY_PEER 88 | end 89 | 90 | @ca_file = @cacert_file 91 | @last_request_time = nil 92 | raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered 93 | 94 | if @formatter_config = conf.elements('format').first 95 | @formatter = formatter_create 96 | end 97 | 98 | if @bulk_request 99 | class << self 100 | alias_method :format, :bulk_request_format 101 | end 102 | @formatter = formatter_create(type: :json) 103 | @serializer = :x_ndjson # secret settings for bulk_request 104 | else 105 | class << self 106 | alias_method :format, :split_request_format 107 | end 108 | end 109 | end 110 | 111 | def start 112 | super 113 | end 114 | 115 | def shutdown 116 | super 117 | end 118 | 119 | def format_url(tag, time, record) 120 | @endpoint_url 121 | end 122 | 123 | def set_body(req, tag, time, record) 124 | if @serializer == :json 125 | set_json_body(req, record) 126 | elsif @serializer == :text 127 | set_text_body(req, record) 128 | elsif @serializer == :raw 129 | set_raw_body(req, record) 130 | elsif @serializer == :x_ndjson 131 | set_bulk_body(req, record) 132 | else 133 | req.set_form_data(record) 134 | end 135 | req 136 | end 137 | 138 | def set_header(req, tag, time, record) 139 | if @custom_headers 140 | @custom_headers.each do |k,v| 141 | req[k] = v 142 | end 143 | req 144 | else 145 | req 146 | end 147 | end 148 | 149 | def compress_body(req, data) 150 | return unless @compress_request 151 | gz = Zlib::GzipWriter.new(StringIO.new) 152 | gz << data 153 | 154 | req['Content-Encoding'] = "gzip" 155 | req.body = gz.close.string 156 | end 157 | 158 | def set_json_body(req, data) 159 | req.body = Yajl.dump(data) 160 | req['Content-Type'] = 'application/json' 161 | compress_body(req, req.body) 162 | end 163 | 164 | def set_text_body(req, data) 165 | req.body = data["message"] 166 | req['Content-Type'] = 'text/plain' 167 | compress_body(req, req.body) 168 | end 169 | 170 | def set_raw_body(req, data) 171 | req.body = data.to_s 172 | req['Content-Type'] = 'application/octet-stream' 173 | compress_body(req, req.body) 174 | end 175 | 176 | def set_bulk_body(req, data) 177 | req.body = data.to_s 178 | req['Content-Type'] = 'application/x-ndjson' 179 | compress_body(req, req.body) 180 | end 181 | 182 | def create_request(tag, time, record) 183 | url = format_url(tag, time, record) 184 | uri = URI.parse(url) 185 | req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.request_uri) 186 | set_body(req, tag, time, record) 187 | set_header(req, tag, time, record) 188 | return req, uri 189 | end 190 | 191 | def http_opts(uri) 192 | opts = { 193 | :use_ssl => uri.scheme == 'https' 194 | } 195 | opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl] 196 | opts[:ca_file] = File.join(@ca_file) if File.file?(@ca_file) 197 | opts[:cert] = OpenSSL::X509::Certificate.new(File.read(@client_cert_path)) if File.file?(@client_cert_path) 198 | opts[:key] = OpenSSL::PKey.read(File.read(@private_key_path), @private_key_passphrase) if File.file?(@private_key_path) 199 | opts 200 | end 201 | 202 | def proxies 203 | ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy'] 204 | end 205 | 206 | def send_request(req, uri) 207 | is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?) 208 | if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec) 209 | log.info('Dropped request due to rate limiting') 210 | return 211 | end 212 | 213 | res = nil 214 | 215 | begin 216 | if @authentication == :basic 217 | req.basic_auth(@username, @password) 218 | elsif @authentication == :bearer 219 | req['authorization'] = "bearer #{@token}" 220 | elsif @authentication == :jwt 221 | req['authorization'] = "jwt #{@token}" 222 | end 223 | @last_request_time = Time.now.to_f 224 | 225 | if proxy = proxies 226 | proxy_uri = URI.parse(proxy) 227 | 228 | res = Net::HTTP.start(uri.host, uri.port, 229 | proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password, 230 | **http_opts(uri)) {|http| http.request(req) } 231 | else 232 | res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) } 233 | end 234 | 235 | rescue => e # rescue all StandardErrors 236 | # server didn't respond 237 | log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'" 238 | raise e if @raise_on_error 239 | else 240 | unless res and res.is_a?(Net::HTTPSuccess) 241 | res_summary = if res 242 | "#{res.code} #{res.message} #{res.body}" 243 | else 244 | "res=nil" 245 | end 246 | if @recoverable_status_codes.include?(res.code.to_i) 247 | raise RecoverableResponse, res_summary 248 | else 249 | log.warn "failed to #{req.method} #{uri} (#{res_summary})" 250 | end 251 | end #end unless 252 | end # end begin 253 | end # end send_request 254 | 255 | def handle_record(tag, time, record) 256 | if @formatter_config 257 | record = @formatter.format(tag, time, record) 258 | end 259 | req, uri = create_request(tag, time, record) 260 | send_request(req, uri) 261 | end 262 | 263 | def handle_records(tag, time, chunk) 264 | req, uri = create_request(tag, time, chunk.read) 265 | send_request(req, uri) 266 | end 267 | 268 | def prefer_buffered_processing 269 | @buffered 270 | end 271 | 272 | def format(tag, time, record) 273 | # For safety. 274 | end 275 | 276 | def split_request_format(tag, time, record) 277 | [time, record].to_msgpack 278 | end 279 | 280 | def bulk_request_format(tag, time, record) 281 | @formatter.format(tag, time, record) 282 | end 283 | 284 | def formatted_to_msgpack_binary? 285 | if @bulk_request 286 | false 287 | else 288 | true 289 | end 290 | end 291 | 292 | def multi_workers_ready? 293 | true 294 | end 295 | 296 | def process(tag, es) 297 | es.each do |time, record| 298 | handle_record(tag, time, record) 299 | end 300 | end 301 | 302 | def write(chunk) 303 | tag = chunk.metadata.tag 304 | @endpoint_url = extract_placeholders(@endpoint_url, chunk) 305 | 306 | log.debug { "#{@http_method.capitalize} data to #{@endpoint_url} with chunk(#{dump_unique_id_hex(chunk.unique_id)})" } 307 | 308 | if @bulk_request 309 | time = Fluent::Engine.now 310 | handle_records(tag, time, chunk) 311 | else 312 | chunk.msgpack_each do |time, record| 313 | handle_record(tag, time, record) 314 | end 315 | end 316 | end 317 | end 318 | -------------------------------------------------------------------------------- /lib/fluent/test/http_output_test.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler' 3 | 4 | begin 5 | Bundler.setup(:default, :development) 6 | rescue Bundler::BundlerError => e 7 | $stderr.puts e.message 8 | $stderr.puts "Run `bundle install` to install missing gems" 9 | exit e.status_code 10 | end 11 | 12 | require 'test/unit' 13 | 14 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 15 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 16 | 17 | require 'fluent/test' 18 | 19 | unless ENV.has_key?('VERBOSE') 20 | nulllogger = Object.new 21 | nulllogger.instance_eval {|obj| 22 | def method_missing(method, *args) 23 | # pass 24 | end 25 | } 26 | $log = nulllogger 27 | end 28 | 29 | class Test::Unit::TestCase 30 | end 31 | 32 | require 'webrick' 33 | require 'webrick/https' 34 | 35 | # to handle POST/PUT/DELETE ... 36 | module WEBrick::HTTPServlet 37 | class ProcHandler < AbstractServlet 38 | alias do_POST do_GET 39 | alias do_PUT do_GET 40 | alias do_DELETE do_GET 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /run_test.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | Dir.glob("test/**/test_*.rb") do |test_rb| 3 | require File.expand_path(test_rb) 4 | end 5 | -------------------------------------------------------------------------------- /test/plugin/script/plugin/formatter_test.rb: -------------------------------------------------------------------------------- 1 | require 'fluent/plugin/formatter' 2 | 3 | module Fluent 4 | module Plugin 5 | class TestFormatter < Formatter 6 | Fluent::Plugin.register_formatter('test', self) 7 | 8 | def configure(conf) 9 | super 10 | end 11 | 12 | def format(tag, time, record) 13 | output = { 14 | "wrapped" => true, 15 | "record" => record 16 | } 17 | output 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /test/plugin/test_out_http.rb: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | require 'net/http' 3 | require 'uri' 4 | require 'yajl' 5 | require 'fluent/test/http_output_test' 6 | require 'fluent/plugin/out_http' 7 | require 'fluent/test/driver/output' 8 | require 'fluent/test/helpers' 9 | require_relative "./script/plugin/formatter_test" 10 | 11 | module OS 12 | # ref. http://stackoverflow.com/questions/170956/how-can-i-find-which-operating-system-my-ruby-program-is-running-on 13 | def OS.windows? 14 | (/cygwin|mswin|mingw|bccwin|wince|emx/ =~ RUBY_PLATFORM) != nil 15 | end 16 | 17 | def OS.mac? 18 | (/darwin/ =~ RUBY_PLATFORM) != nil 19 | end 20 | 21 | def OS.unix? 22 | !OS.windows? 23 | end 24 | 25 | def OS.linux? 26 | OS.unix? and not OS.mac? 27 | end 28 | end 29 | 30 | class HTTPOutputTestBase < Test::Unit::TestCase 31 | include Fluent::Test::Helpers 32 | 33 | def self.port 34 | 5126 35 | end 36 | 37 | def self.server_config 38 | config = {BindAddress: '127.0.0.1', Port: port} 39 | if ENV['VERBOSE'] 40 | logger = WEBrick::Log.new(STDOUT, WEBrick::BasicLog::DEBUG) 41 | config[:Logger] = logger 42 | config[:AccessLog] = [] 43 | end 44 | config 45 | end 46 | 47 | def self.test_http_client(**opts) 48 | opts = opts.merge(open_timeout: 1, read_timeout: 1) 49 | Net::HTTP.start('127.0.0.1', port, **opts) 50 | end 51 | 52 | # setup / teardown for servers 53 | def setup 54 | Fluent::Test.setup 55 | @posts = [] 56 | @puts = [] 57 | @prohibited = 0 58 | @requests = 0 59 | @auth = false 60 | @headers = {} 61 | @dummy_server_thread = Thread.new do 62 | srv = WEBrick::HTTPServer.new(self.class.server_config) 63 | begin 64 | allowed_methods = %w(POST PUT) 65 | srv.mount_proc('/api') { |req,res| 66 | @requests += 1 67 | unless allowed_methods.include? req.request_method 68 | res.status = 405 69 | res.body = 'request method mismatch' 70 | next 71 | end 72 | req.each do |key, value| 73 | @headers[key] = value 74 | end 75 | if @auth and req.header['authorization'][0] == 'Basic YWxpY2U6c2VjcmV0IQ==' # pattern of user='alice' passwd='secret!' 76 | # ok, authorized 77 | # pattern of bear #{Base64.encode64('secret token!')} 78 | elsif @auth and req.header['authorization'][0] == 'bearer c2VjcmV0IHRva2VuIQ==' 79 | # pattern of jwt 80 | # header: { 81 | # "alg": "HS256", 82 | # "typ": "JWT" 83 | # } 84 | # payload: { 85 | # "iss": "Hoge Publisher", 86 | # "sub": "Hoge User" 87 | # } 88 | # signature: 89 | # HS256(base64UrlEncode(header) + "." + 90 | # base64UrlEncode(payload) + "." + 91 | # secret) 92 | elsif @auth and req.header['authorization'][0] == 'jwt eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJIb2dlIFB1Ymxpc2hlciIsInN1YiI6IkhvZ2UgVXNlciJ9.V2NL7YgCWNt5d3vTXFrcRLpRImO2cU2JZ4mQglqw3rE' 93 | elsif @auth 94 | res.status = 403 95 | @prohibited += 1 96 | next 97 | else 98 | # ok, authorization not required 99 | end 100 | 101 | expander = -> (req) { 102 | if req["Content-Encoding"] == "gzip" 103 | StringIO.open(req.body, 'rb'){|sio| 104 | Zlib::GzipReader.wrap(sio).read 105 | } 106 | else 107 | req.body 108 | end 109 | } 110 | 111 | record = {:auth => nil} 112 | if req.content_type == 'application/json' 113 | record[:json] = Yajl.load(expander.call(req)) 114 | elsif req.content_type == 'text/plain' 115 | puts req 116 | record[:data] = expander.call(req) 117 | elsif req.content_type == 'application/octet-stream' 118 | record[:data] = expander.call(req) 119 | elsif req.content_type == 'application/x-ndjson' 120 | data = [] 121 | expander.call(req).each_line { |l| 122 | data << Yajl.load(l) 123 | } 124 | record[:x_ndjson] = data 125 | else 126 | record[:form] = Hash[*(req.body.split('&').map{|kv|kv.split('=')}.flatten)] 127 | end 128 | 129 | instance_variable_get("@#{req.request_method.downcase}s").push(record) 130 | 131 | res.status = 200 132 | } 133 | srv.mount_proc('/modified-api') { |req,res| 134 | res.status = 303 135 | res.body = 'See other' 136 | } 137 | srv.mount_proc('/') { |req,res| 138 | res.status = 200 139 | res.body = 'running' 140 | } 141 | srv.start 142 | ensure 143 | srv.shutdown 144 | end 145 | end 146 | 147 | # to wait completion of dummy server.start() 148 | require 'thread' 149 | cv = ConditionVariable.new 150 | watcher = Thread.new { 151 | connected = false 152 | while not connected 153 | begin 154 | client = self.class.test_http_client 155 | client.request_get('/') 156 | connected = true 157 | rescue Errno::ECONNREFUSED 158 | sleep 0.1 159 | rescue StandardError => e 160 | p e 161 | sleep 0.1 162 | end 163 | end 164 | cv.signal 165 | } 166 | mutex = Mutex.new 167 | mutex.synchronize { 168 | cv.wait(mutex) 169 | } 170 | end 171 | 172 | def test_dummy_server 173 | client = self.class.test_http_client 174 | post_header = { 'Content-Type' => 'application/x-www-form-urlencoded' } 175 | 176 | assert_equal '200', client.request_get('/').code 177 | assert_equal '200', client.request_post('/api/service/metrics/hoge', 'number=1&mode=gauge', post_header).code 178 | 179 | assert_equal 1, @posts.size 180 | 181 | assert_equal '1', @posts[0][:form]['number'] 182 | assert_equal 'gauge', @posts[0][:form]['mode'] 183 | assert_nil @posts[0][:auth] 184 | 185 | assert_equal '303', client.request_get('/modified-api').code 186 | 187 | @auth = true 188 | 189 | assert_equal '403', client.request_post('/api/service/metrics/pos', 'number=30&mode=gauge', post_header).code 190 | 191 | req_with_auth = lambda do |number, mode, user, pass| 192 | req = Net::HTTP::Post.new("/api/service/metrics/pos") 193 | req.content_type = 'application/x-www-form-urlencoded' 194 | req.basic_auth user, pass 195 | req.set_form_data({'number'=>number, 'mode'=>mode}) 196 | req 197 | end 198 | 199 | assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code 200 | 201 | assert_equal '403', client.request(req_with_auth.call(500, 'count', 'alice', 'wrong password!')).code 202 | 203 | assert_equal 1, @posts.size 204 | 205 | assert_equal '200', client.request(req_with_auth.call(500, 'count', 'alice', 'secret!')).code 206 | 207 | assert_equal 2, @posts.size 208 | 209 | end 210 | 211 | def teardown 212 | @dummy_server_thread.kill 213 | @dummy_server_thread.join 214 | end 215 | 216 | def create_driver(conf) 217 | Fluent::Test::Driver::Output.new(Fluent::Plugin::HTTPOutput).configure(conf) 218 | end 219 | end 220 | 221 | class HTTPOutputTest < HTTPOutputTestBase 222 | CONFIG = %[ 223 | endpoint_url http://127.0.0.1:#{port}/api/ 224 | ] 225 | 226 | CONFIG_QUERY_PARAM = %[ 227 | endpoint_url http://127.0.0.1:#{port}/api?foo=bar&baz=qux 228 | ] 229 | 230 | CONFIG_JSON = %[ 231 | endpoint_url http://127.0.0.1:#{port}/api/ 232 | serializer json 233 | ] 234 | 235 | CONFIG_TEXT = %[ 236 | endpoint_url http://127.0.0.1:#{port}/api/ 237 | serializer text 238 | ] 239 | 240 | CONFIG_RAW = %[ 241 | endpoint_url http://127.0.0.1:#{port}/api/ 242 | serializer raw 243 | ] 244 | 245 | CONFIG_PUT = %[ 246 | endpoint_url http://127.0.0.1:#{port}/api/ 247 | http_method put 248 | ] 249 | 250 | CONFIG_HTTP_ERROR = %[ 251 | endpoint_url https://127.0.0.1:#{port - 1}/api/ 252 | ] 253 | 254 | CONFIG_HTTP_ERROR_SUPPRESSED = %[ 255 | endpoint_url https://127.0.0.1:#{port - 1}/api/ 256 | raise_on_error false 257 | ] 258 | 259 | RATE_LIMIT_MSEC = 1200 260 | 261 | CONFIG_RATE_LIMIT = %[ 262 | endpoint_url http://127.0.0.1:#{port}/api/ 263 | rate_limit_msec #{RATE_LIMIT_MSEC} 264 | ] 265 | 266 | def test_configure 267 | d = create_driver CONFIG 268 | assert_equal "http://127.0.0.1:#{self.class.port}/api/", d.instance.endpoint_url 269 | assert_equal :form, d.instance.serializer 270 | assert_equal [503], d.instance.recoverable_status_codes 271 | 272 | d = create_driver CONFIG_JSON 273 | assert_equal "http://127.0.0.1:#{self.class.port}/api/", d.instance.endpoint_url 274 | assert_equal :json, d.instance.serializer 275 | end 276 | 277 | test 'lack of tag in chunk_keys' do 278 | assert_raise_message(/'tag' in chunk_keys is required./) do 279 | create_driver(Fluent::Config::Element.new( 280 | 'ROOT', '', { 281 | '@type' => 'http', 282 | 'endpoint_url' => "http://127.0.0.1:#{self.class.port}/api/", 283 | 'buffered' => true, 284 | }, [ 285 | Fluent::Config::Element.new('buffer', 'mykey', { 286 | 'chunk_keys' => 'mykey' 287 | }, []) 288 | ] 289 | )) 290 | end 291 | end 292 | 293 | def test_emit_form 294 | d = create_driver CONFIG 295 | d.run(default_tag: 'test.metrics') do 296 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") }) 297 | end 298 | 299 | assert_equal 1, @posts.size 300 | record = @posts[0] 301 | 302 | assert_equal '50', record[:form]['field1'] 303 | assert_equal '20', record[:form]['field2'] 304 | assert_equal '10', record[:form]['field3'] 305 | assert_equal '1', record[:form]['otherfield'] 306 | assert_equal URI.encode_www_form_component("あ").upcase, record[:form]['binary'].upcase 307 | assert_nil record[:auth] 308 | 309 | d.run(default_tag: 'test.metrics') do 310 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 }) 311 | end 312 | 313 | assert_equal 2, @posts.size 314 | end 315 | 316 | def test_emit_form_with_query_params 317 | d = create_driver CONFIG_QUERY_PARAM 318 | d.run(default_tag: 'test.metrics') do 319 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") }) 320 | end 321 | 322 | assert_equal 1, @posts.size 323 | record = @posts[0] 324 | 325 | assert_equal '50', record[:form]['field1'] 326 | assert_equal '20', record[:form]['field2'] 327 | assert_equal '10', record[:form]['field3'] 328 | assert_equal '1', record[:form]['otherfield'] 329 | assert_equal URI.encode_www_form_component("あ").upcase, record[:form]['binary'].upcase 330 | assert_nil record[:auth] 331 | 332 | d.run(default_tag: 'test.metrics') do 333 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 }) 334 | end 335 | 336 | assert_equal 2, @posts.size 337 | end 338 | 339 | def test_emit_form_with_custom_headers 340 | d = create_driver CONFIG + %[custom_headers {"key":"custom","token":"arbitrary"}] 341 | d.run(default_tag: 'test.metrics') do 342 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") }) 343 | end 344 | 345 | assert_true @headers.has_key?("key") 346 | assert_equal "custom", @headers["key"] 347 | assert_true @headers.has_key?("token") 348 | assert_equal "arbitrary", @headers["token"] 349 | 350 | assert_equal 1, @posts.size 351 | record = @posts[0] 352 | 353 | assert_equal '50', record[:form]['field1'] 354 | assert_equal '20', record[:form]['field2'] 355 | assert_equal '10', record[:form]['field3'] 356 | assert_equal '1', record[:form]['otherfield'] 357 | assert_equal URI.encode_www_form_component("あ").upcase, record[:form]['binary'].upcase 358 | assert_nil record[:auth] 359 | 360 | d.run(default_tag: 'test.metrics') do 361 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 }) 362 | end 363 | 364 | assert_equal 2, @posts.size 365 | end 366 | 367 | class BufferedEmitTest < self 368 | def test_emit_form 369 | d = create_driver CONFIG + %[buffered true] 370 | d.run(default_tag: 'test.metrics', shutdown: false) do 371 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit") }) 372 | end 373 | 374 | assert_equal 1, @posts.size 375 | record = @posts[0] 376 | 377 | assert_equal '50', record[:form]['field1'] 378 | assert_equal '20', record[:form]['field2'] 379 | assert_equal '10', record[:form]['field3'] 380 | assert_equal '1', record[:form]['otherfield'] 381 | assert_equal URI.encode_www_form_component("あ").upcase, record[:form]['binary'].upcase 382 | assert_nil record[:auth] 383 | 384 | d.run(default_tag: 'test.metrics', shutdown: false) do 385 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 }) 386 | end 387 | 388 | assert_equal 2, @posts.size 389 | end 390 | 391 | def test_emit_form_with_placeholders 392 | d = create_driver(Fluent::Config::Element.new( 393 | 'ROOT', '' , 394 | {"endpoint_url" => "${endpoint}", 395 | "buffered" => true}, 396 | [Fluent::Config::Element.new('buffer', 'tag, endpoint', {"@type" => "memory"} ,[])])) 397 | 398 | d.run(default_tag: 'test.metrics', shutdown: false) do 399 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => "\xe3\x81\x82".force_encoding("ascii-8bit"), 'endpoint' => "http://127.0.0.1:#{self.class.port}/modified-api/" }) 400 | end 401 | 402 | assert_equal 0, @posts.size # post into other URI 403 | assert_equal "http://127.0.0.1:#{self.class.port}/modified-api/", d.instance.endpoint_url 404 | end 405 | 406 | def test_emit_form_put 407 | d = create_driver CONFIG_PUT + %[buffered true] 408 | d.run(default_tag: 'test.metrics', shutdown: false) do 409 | d.feed({ 'field1' => 50 }) 410 | end 411 | 412 | assert_equal 0, @posts.size 413 | assert_equal 1, @puts.size 414 | record = @puts[0] 415 | 416 | assert_equal '50', record[:form]['field1'] 417 | assert_nil record[:auth] 418 | 419 | d.run(default_tag: 'test.metrics', shutdown: false) do 420 | d.feed({ 'field1' => 50 }) 421 | end 422 | 423 | assert_equal 0, @posts.size 424 | assert_equal 2, @puts.size 425 | end 426 | 427 | def test_emit_json 428 | binary_string = "\xe3\x81\x82" 429 | d = create_driver CONFIG_JSON + %[buffered true] 430 | d.run(default_tag: 'test.metrics') do 431 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string }) 432 | end 433 | 434 | assert_equal 1, @posts.size 435 | record = @posts[0] 436 | 437 | assert_equal 50, record[:json]['field1'] 438 | assert_equal 20, record[:json]['field2'] 439 | assert_equal 10, record[:json]['field3'] 440 | assert_equal 1, record[:json]['otherfield'] 441 | assert_equal binary_string, record[:json]['binary'] 442 | assert_nil record[:auth] 443 | end 444 | 445 | def test_emit_json_with_compression 446 | binary_string = "\xe3\x81\x82" 447 | d = create_driver CONFIG_JSON + %[buffered true\ncompress_request true] 448 | d.run(default_tag: 'test.metrics') do 449 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string }) 450 | end 451 | 452 | assert_equal 1, @posts.size 453 | record = @posts[0] 454 | 455 | assert_equal 50, record[:json]['field1'] 456 | assert_equal 20, record[:json]['field2'] 457 | assert_equal 10, record[:json]['field3'] 458 | assert_equal 1, record[:json]['otherfield'] 459 | assert_equal binary_string, record[:json]['binary'] 460 | assert_nil record[:auth] 461 | end 462 | 463 | def test_emit_x_ndjson 464 | binary_string = "\xe3\x81\x82" 465 | d = create_driver CONFIG_JSON + %[buffered true\nbulk_request] 466 | d.run(default_tag: 'test.metrics') do 467 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string }) 468 | d.feed({ 'field1' => 70, 'field2' => 30, 'field3' => 20, 'otherfield' => 2, 'binary' => binary_string }) 469 | end 470 | 471 | assert_equal 1, @posts.size 472 | record = @posts[0] 473 | 474 | expected =[ 475 | { 476 | "binary" => "\u3042", 477 | "field1" => 50, 478 | "field2" => 20, 479 | "field3" => 10, 480 | "otherfield" => 1 481 | }, 482 | { 483 | "binary" => "\u3042", 484 | "field1" => 70, 485 | "field2" => 30, 486 | "field3" => 20, 487 | "otherfield" => 2 488 | } 489 | ] 490 | 491 | assert_equal expected, record[:x_ndjson] 492 | assert_nil record[:auth] 493 | end 494 | 495 | def test_emit_x_ndjson_with_compression 496 | binary_string = "\xe3\x81\x82" 497 | d = create_driver CONFIG_JSON + %[buffered true\nbulk_request true\ncompress_request true] 498 | d.run(default_tag: 'test.metrics') do 499 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string }) 500 | d.feed({ 'field1' => 70, 'field2' => 30, 'field3' => 20, 'otherfield' => 2, 'binary' => binary_string }) 501 | end 502 | 503 | assert_equal 1, @posts.size 504 | record = @posts[0] 505 | 506 | expected =[ 507 | { 508 | "binary" => "\u3042", 509 | "field1" => 50, 510 | "field2" => 20, 511 | "field3" => 10, 512 | "otherfield" => 1 513 | }, 514 | { 515 | "binary" => "\u3042", 516 | "field1" => 70, 517 | "field2" => 30, 518 | "field3" => 20, 519 | "otherfield" => 2 520 | } 521 | ] 522 | 523 | assert_equal expected, record[:x_ndjson] 524 | assert_nil record[:auth] 525 | end 526 | end 527 | 528 | def test_emit_form_put 529 | d = create_driver CONFIG_PUT 530 | d.run(default_tag: 'test.metrics') do 531 | d.feed({ 'field1' => 50 }) 532 | end 533 | 534 | assert_equal 0, @posts.size 535 | assert_equal 1, @puts.size 536 | record = @puts[0] 537 | 538 | assert_equal '50', record[:form]['field1'] 539 | assert_nil record[:auth] 540 | 541 | d.run(default_tag: 'test.metrics') do 542 | d.feed({ 'field1' => 50 }) 543 | end 544 | 545 | assert_equal 0, @posts.size 546 | assert_equal 2, @puts.size 547 | end 548 | 549 | def test_emit_json 550 | binary_string = "\xe3\x81\x82" 551 | d = create_driver CONFIG_JSON 552 | d.run(default_tag: 'test.metrics') do 553 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string }) 554 | end 555 | 556 | assert_equal 1, @posts.size 557 | record = @posts[0] 558 | 559 | assert_equal 50, record[:json]['field1'] 560 | assert_equal 20, record[:json]['field2'] 561 | assert_equal 10, record[:json]['field3'] 562 | assert_equal 1, record[:json]['otherfield'] 563 | assert_equal binary_string, record[:json]['binary'] 564 | assert_nil record[:auth] 565 | end 566 | 567 | def test_emit_json_with_compression 568 | binary_string = "\xe3\x81\x82" 569 | d = create_driver CONFIG_JSON + %[compress_request true] 570 | d.run(default_tag: 'test.metrics') do 571 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string }) 572 | end 573 | 574 | assert_equal 1, @posts.size 575 | record = @posts[0] 576 | 577 | assert_equal 50, record[:json]['field1'] 578 | assert_equal 20, record[:json]['field2'] 579 | assert_equal 10, record[:json]['field3'] 580 | assert_equal 1, record[:json]['otherfield'] 581 | assert_equal binary_string, record[:json]['binary'] 582 | assert_nil record[:auth] 583 | end 584 | 585 | def test_emit_text 586 | binary_string = "\xe3\x81\x82" 587 | d = create_driver CONFIG_TEXT 588 | d.run(default_tag: 'test.metrics') do 589 | d.feed({ "message" => "hello" }) 590 | end 591 | assert_equal 1, @posts.size 592 | record = @posts[0] 593 | assert_equal 'hello', record[:data] 594 | assert_nil record[:auth] 595 | end 596 | 597 | def test_emit_text_with_compression 598 | d = create_driver CONFIG_TEXT + %[compress_request true] 599 | d.run(default_tag: 'test.metrics') do 600 | d.feed({ "message" => "hello" }) 601 | end 602 | assert_equal 1, @posts.size 603 | record = @posts[0] 604 | assert_equal 'hello', record[:data] 605 | assert_nil record[:auth] 606 | end 607 | 608 | def test_emit_raw 609 | binary_string = "\xe3\x81\x82" 610 | d = create_driver CONFIG_RAW + %[format msgpack] 611 | d.run(default_tag: 'test.metrics') do 612 | d.feed({ "message" => "hello" }) 613 | end 614 | assert_equal 1, @posts.size 615 | record = @posts[0] 616 | assert_equal ({ "message" => "hello" }).to_msgpack, record[:data] 617 | assert_nil record[:auth] 618 | end 619 | 620 | def test_emit_raw_with_compression 621 | binary_string = "\xe3\x81\x82" 622 | d = create_driver CONFIG_RAW + %[format msgpack\ncompress_request true] 623 | d.run(default_tag: 'test.metrics') do 624 | d.feed({ "message" => "hello" }) 625 | end 626 | assert_equal 1, @posts.size 627 | record = @posts[0] 628 | assert_equal ({ "message" => "hello" }).to_msgpack, record[:data].force_encoding("ascii-8bit") 629 | assert_nil record[:auth] 630 | end 631 | 632 | def test_http_error_is_raised 633 | d = create_driver CONFIG_HTTP_ERROR 634 | assert_raise Errno::ECONNREFUSED do 635 | d.run(default_tag: 'test.metrics') do 636 | d.feed({ 'field1' => 50 }) 637 | end 638 | end 639 | end 640 | 641 | def test_http_error_is_suppressed_with_raise_on_error_false 642 | d = create_driver CONFIG_HTTP_ERROR_SUPPRESSED 643 | d.run(default_tag: 'test.metrics') do 644 | d.feed({ 'field1' => 50 }) 645 | end 646 | # drive asserts the next output chain is called; 647 | # so no exception means our plugin handled the error 648 | 649 | assert_equal 0, @requests 650 | end 651 | 652 | def test_rate_limiting 653 | d = create_driver CONFIG_RATE_LIMIT 654 | record = { :k => 1 } 655 | 656 | last_emit = _current_msec 657 | d.run(default_tag: 'test.metrics') do 658 | d.feed(record) 659 | end 660 | 661 | assert_equal 1, @posts.size 662 | 663 | d.run(default_tag: 'test.metrics') do 664 | d.feed({}) 665 | end 666 | assert last_emit + RATE_LIMIT_MSEC > _current_msec, "Still under rate limiting interval" 667 | assert_equal 1, @posts.size 668 | 669 | wait_msec = 500 670 | sleep (last_emit + RATE_LIMIT_MSEC - _current_msec + wait_msec) * 0.001 671 | 672 | assert last_emit + RATE_LIMIT_MSEC < _current_msec, "No longer under rate limiting interval" 673 | d.run(default_tag: 'test.metrics') do 674 | d.feed(record) 675 | end 676 | assert_equal 2, @posts.size 677 | end 678 | 679 | def _current_msec 680 | Time.now.to_f * 1000 681 | end 682 | 683 | def test_auth 684 | @auth = true # enable authentication of dummy server 685 | 686 | d = create_driver(CONFIG) 687 | d.run(default_tag: 'test.metrics') do 688 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 }) 689 | end # failed in background, and output warn log 690 | 691 | assert_equal 0, @posts.size 692 | assert_equal 1, @prohibited 693 | 694 | d = create_driver(CONFIG + %[ 695 | authentication basic 696 | username alice 697 | password wrong_password 698 | ]) 699 | d.run(default_tag: 'test.metrics') do 700 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 }) 701 | end # failed in background, and output warn log 702 | 703 | assert_equal 0, @posts.size 704 | assert_equal 2, @prohibited 705 | 706 | d = create_driver(CONFIG + %[ 707 | authentication basic 708 | username alice 709 | password secret! 710 | ]) 711 | d.run(default_tag: 'test.metrics') do 712 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 }) 713 | end # failed in background, and output warn log 714 | 715 | assert_equal 1, @posts.size 716 | assert_equal 2, @prohibited 717 | 718 | require 'base64' 719 | d = create_driver(CONFIG + %[ 720 | authentication bearer 721 | token #{Base64.encode64('secret token!')} 722 | ]) 723 | d.run(default_tag: 'test.metrics') do 724 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 }) 725 | end # failed in background, and output warn log 726 | 727 | assert_equal 2, @posts.size 728 | assert_equal 2, @prohibited 729 | 730 | d = create_driver(CONFIG + %[ 731 | authentication jwt 732 | token eyJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJIb2dlIFB1Ymxpc2hlciIsInN1YiI6IkhvZ2UgVXNlciJ9.V2NL7YgCWNt5d3vTXFrcRLpRImO2cU2JZ4mQglqw3rE 733 | ]) 734 | d.run(default_tag: 'test.metrics') do 735 | d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1 }) 736 | end # failed in background, and output warn log 737 | 738 | assert_equal 3, @posts.size 739 | assert_equal 2, @prohibited 740 | end 741 | 742 | class CustomFormatterTest < self 743 | def test_new_config 744 | config = Fluent::Config::Element.new( 745 | 'ROOT', '', 746 | {"@type" => "http", 747 | "endpoint_url" => "http://127.0.0.1:#{self.class.port}/api/", 748 | "serializer" => "json"}, [ 749 | Fluent::Config::Element.new('format', '', { 750 | "@type" => "test" 751 | }, []) 752 | ]) 753 | d = create_driver config 754 | payload = {"field" => 1} 755 | d.run(default_tag: 'test.metrics') do 756 | d.feed(payload) 757 | end 758 | 759 | record = @posts[0] 760 | expected = {"wrapped" => true, "record" => payload} 761 | assert_equal expected, record[:json] 762 | end 763 | 764 | def test_legacy_config 765 | config = %[ 766 | endpoint_url http://127.0.0.1:#{self.class.port}/api/ 767 | serializer json 768 | format test 769 | ] 770 | 771 | d = create_driver config 772 | payload = {"field" => 1} 773 | d.run(default_tag: 'test.metrics') do 774 | d.feed(payload) 775 | end 776 | 777 | record = @posts[0] 778 | expected = {"wrapped" => true, "record" => payload} 779 | assert_equal expected, record[:json] 780 | end 781 | end 782 | end 783 | 784 | class HTTPSOutputTest < HTTPOutputTestBase 785 | def self.port 786 | 5127 787 | end 788 | 789 | def self.server_config 790 | config = super 791 | config[:SSLEnable] = true 792 | config[:SSLCertName] = [["CN", WEBrick::Utils::getservername]] 793 | config 794 | end 795 | 796 | def self.test_http_client 797 | super( 798 | use_ssl: true, 799 | verify_mode: OpenSSL::SSL::VERIFY_NONE, 800 | ) 801 | end 802 | 803 | def test_configure 804 | test_uri = URI.parse("https://127.0.0.1/") 805 | 806 | ssl_config = %[ 807 | endpoint_url https://127.0.0.1:#{self.class.port}/api/ 808 | ] 809 | d = create_driver ssl_config 810 | expected_endpoint_url = "https://127.0.0.1:#{self.class.port}/api/" 811 | assert_equal expected_endpoint_url, d.instance.endpoint_url 812 | http_opts = d.instance.http_opts(test_uri) 813 | assert_equal true, http_opts[:use_ssl] 814 | assert_equal OpenSSL::SSL::VERIFY_PEER, http_opts[:verify_mode] 815 | 816 | no_verify_config = %[ 817 | endpoint_url https://127.0.0.1:#{self.class.port}/api/ 818 | ssl_no_verify true 819 | ] 820 | d = create_driver no_verify_config 821 | http_opts = d.instance.http_opts(test_uri) 822 | assert_equal true, http_opts[:use_ssl] 823 | assert_equal OpenSSL::SSL::VERIFY_NONE, http_opts[:verify_mode] 824 | 825 | cacert_file_config = %[ 826 | endpoint_url https://127.0.0.1:#{self.class.port}/api/ 827 | ssl_no_verify true 828 | cacert_file /tmp/ssl.cert 829 | ] 830 | d = create_driver cacert_file_config 831 | FileUtils::touch '/tmp/ssl.cert' 832 | http_opts = d.instance.http_opts(test_uri) 833 | assert_equal true, http_opts[:use_ssl] 834 | assert_equal OpenSSL::SSL::VERIFY_NONE, http_opts[:verify_mode] 835 | assert_equal true, File.file?('/tmp/ssl.cert') 836 | puts http_opts 837 | assert_equal File.join('/tmp/ssl.cert'), http_opts[:ca_file] 838 | end 839 | 840 | def test_emit_form_ssl 841 | config = %[ 842 | endpoint_url https://127.0.0.1:#{self.class.port}/api/ 843 | ssl_no_verify true 844 | ] 845 | d = create_driver config 846 | d.run(default_tag: 'test.metrics') do 847 | d.feed({ 'field1' => 50 }) 848 | end 849 | 850 | assert_equal 1, @posts.size 851 | record = @posts[0] 852 | 853 | assert_equal '50', record[:form]['field1'] 854 | end 855 | 856 | def test_emit_form_ssl_ca 857 | config = %[ 858 | endpoint_url https://127.0.0.1:#{self.class.port}/api/ 859 | ssl_no_verify true 860 | cacert_file /tmp/ssl.cert 861 | ] 862 | d = create_driver config 863 | d.run(default_tag: 'test.metrics') do 864 | d.feed({ 'field1' => 50 }) 865 | end 866 | 867 | assert_equal 1, @posts.size 868 | record = @posts[0] 869 | 870 | assert_equal '50', record[:form]['field1'] 871 | end 872 | end 873 | --------------------------------------------------------------------------------