├── .coveralls.yml ├── .gitignore ├── .rspec ├── .simplecov ├── .travis.yml ├── Appraisals ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── fluent-plugin-elb-access-log.gemspec ├── gemfiles ├── fluentd_0.12.gemfile └── fluentd_0.14.gemfile ├── lib ├── fluent │ └── plugin │ │ └── in_elb_access_log.rb └── fluent_plugin_elb_access_log │ └── version.rb └── spec ├── in_elb_access_log_alb_spec.rb ├── in_elb_access_log_clb_spec.rb ├── in_elb_access_log_client_spec.rb ├── in_elb_access_log_config_spec.rb └── spec_helper.rb /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: travis-ci 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | *.bundle 11 | *.so 12 | *.o 13 | *.a 14 | mkmf.log 15 | test.rb 16 | /gemfiles/*.lock 17 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper 2 | --colour 3 | -------------------------------------------------------------------------------- /.simplecov: -------------------------------------------------------------------------------- 1 | SimpleCov.start do 2 | # exclude directories and files 3 | add_filter "/spec/" 4 | end 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.1.10 4 | - 2.2.9 5 | - 2.3.6 6 | - 2.4.3 7 | - 2.5.0 8 | before_install: 9 | - gem update --system 10 | script: 11 | - bundle install 12 | - bundle exec rake 13 | gemfile: 14 | - gemfiles/fluentd_0.12.gemfile 15 | - gemfiles/fluentd_0.14.gemfile 16 | -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | appraise 'fluentd-0.12' do 2 | gem 'fluentd', '>= 0.12', '< 0.14' 3 | end 4 | 5 | appraise 'fluentd-0.14' do 6 | gem 'fluentd', '>= 0.14' 7 | end 8 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in fluent-plugin-elb-access-log.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Genki Sugawara 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fluent-plugin-elb-access-log 2 | 3 | Fluentd input plugin for AWS ELB Access Logs. 4 | 5 | [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elb-access-log.svg)](http://badge.fury.io/rb/fluent-plugin-elb-access-log) 6 | [![Build Status](https://travis-ci.org/winebarrel/fluent-plugin-elb-access-log.svg?branch=master)](https://travis-ci.org/winebarrel/fluent-plugin-elb-access-log) 7 | [![Coverage Status](https://coveralls.io/repos/github/winebarrel/fluent-plugin-elb-access-log/badge.svg?branch=master)](https://coveralls.io/github/winebarrel/fluent-plugin-elb-access-log?branch=master) 8 | 9 | ## Installation 10 | 11 | Add this line to your application's Gemfile: 12 | 13 | ```ruby 14 | gem 'fluent-plugin-elb-access-log' 15 | ``` 16 | 17 | And then execute: 18 | 19 | $ bundle 20 | 21 | Or install it yourself as: 22 | 23 | $ gem install fluent-plugin-elb-access-log 24 | 25 | ## Configuration 26 | 27 | ```apache 28 | 29 | @type elb_access_log 30 | #aws_key_id YOUR_ACCESS_KEY_ID 31 | #aws_sec_key YOUR_SECRET_ACCESS_KEY 32 | #profile PROFILE_NAME 33 | #credentials_path path/to/credentials_file 34 | #http_proxy http://... 35 | 36 | account_id 123456789012 # required 37 | region us-west-1 # required 38 | s3_bucket BUCKET_NAME # required 39 | #s3_prefix PREFIX 40 | 41 | #tag elb.access_log 42 | #tsfile_path /var/tmp/fluent-plugin-elb-access-log.ts 43 | #histfile_path /var/tmp/fluent-plugin-elb-access-log.history 44 | #interval 300 45 | #start_datetime 2015/05/24 17:00 46 | #buffer_sec 600 47 | #history_length 100 48 | #sampling_interval 1 49 | #debug false 50 | #elb_type clb # or alb 51 | #filter elb_status_code:^2,timestamp:^2018 52 | #filter_operator and # or "or" 53 | #type_cast true 54 | #parse_request true 55 | #split_addr_port true 56 | #file_filter REGEXP 57 | #request_separator . 58 | 59 | ``` 60 | 61 | ## Outout 62 | 63 | ### CLB 64 | 65 | see http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html 66 | 67 | ```json 68 | { 69 | "timestamp":"2015-05-24T08:25:36.229576Z", 70 | "elb":"hoge", 71 | "client":"14.14.124.20", 72 | "client_port":52232, 73 | "backend":"10.0.199.184", 74 | "backend_port":80, 75 | "request_processing_time":5.5e-05, 76 | "backend_processing_time":0.000893, 77 | "response_processing_time":5.7e-05, 78 | "elb_status_code":200, 79 | "backend_status_code":200, 80 | "received_bytes":0, 81 | "sent_bytes":3, 82 | "request":"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 83 | "user_agent":"curl/7.30.0", 84 | "ssl_cipher":"-", 85 | "ssl_protocol":"-", 86 | "request.method":"GET", 87 | "request.uri":"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 88 | "request.http_version":"HTTP/1.1", 89 | "request.uri.scheme":"http", 90 | "request.uri.user":null, 91 | "request.uri.host":"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 92 | "request.uri.port":80, 93 | "request.uri.path":"/", 94 | "request.uri.query":null, 95 | "request.uri.fragment":null 96 | } 97 | ``` 98 | 99 | ### ALB 100 | 101 | see http://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html 102 | 103 | ```json 104 | { 105 | "type": "https", 106 | "timestamp": "2015-05-24T19:55:36.000000Z", 107 | "elb": "hoge", 108 | "client_port": 57673, 109 | "target_port": 80, 110 | "request_processing_time": 5.3e-05, 111 | "target_processing_time": 0.000913, 112 | "response_processing_time": 3.6e-05, 113 | "elb_status_code": 200, 114 | "target_status_code": 200, 115 | "received_bytes": 0, 116 | "sent_bytes": 3, 117 | "request": "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 118 | "user_agent": "curl/7.30.0", 119 | "ssl_cipher": "ssl_cipher", 120 | "ssl_protocol": "ssl_protocol", 121 | "target_group_arn": "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 122 | "trace_id": "Root=xxx", 123 | "domain_name": "-", 124 | "chosen_cert_arn": "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 125 | "client": "14.14.124.20", 126 | "target": "10.0.199.184", 127 | "request.method": "GET", 128 | "request.uri": "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 129 | "request.http_version": "HTTP/1.1", 130 | "request.uri.scheme": "http", 131 | "request.uri.user": null, 132 | "request.uri.host": "hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 133 | "request.uri.port": 80, 134 | "request.uri.path": "/", 135 | "request.uri.query": null, 136 | "request.uri.fragment": null 137 | } 138 | ``` 139 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | require 'rspec/core/rake_task' 3 | 4 | RSpec::Core::RakeTask.new('spec') 5 | 6 | task :default => :spec 7 | -------------------------------------------------------------------------------- /fluent-plugin-elb-access-log.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'fluent_plugin_elb_access_log/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = 'fluent-plugin-elb-access-log' 8 | spec.version = FluentPluginElbAccessLog::VERSION 9 | spec.authors = ['Genki Sugawara'] 10 | spec.email = ['sugawara@cookpad.com'] 11 | spec.summary = %q{Fluentd input plugin for AWS ELB Access Logs.} 12 | spec.description = %q{Fluentd input plugin for AWS ELB Access Logs.} 13 | spec.homepage = 'https://github.com/winebarrel/fluent-plugin-elb-access-log' 14 | spec.license = 'MIT' 15 | 16 | spec.files = `git ls-files -z`.split("\x0") 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ['lib'] 20 | 21 | spec.add_dependency 'fluentd' 22 | spec.add_dependency 'aws-sdk-s3', '~> 1.8' 23 | spec.add_dependency 'addressable' 24 | spec.add_dependency 'multiple_files_gzip_reader' 25 | spec.add_development_dependency 'bundler' 26 | spec.add_development_dependency 'rake' 27 | spec.add_development_dependency 'rspec', '>= 3.0.0' 28 | spec.add_development_dependency 'timecop' 29 | spec.add_development_dependency 'test-unit', '>= 3.1.0' 30 | spec.add_development_dependency 'rspec-match_table', '>= 0.1.1' 31 | spec.add_development_dependency 'coveralls' 32 | spec.add_development_dependency 'appraisal', '>= 2.2' 33 | end 34 | -------------------------------------------------------------------------------- /gemfiles/fluentd_0.12.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "fluentd", ">= 0.12", "< 0.14" 6 | 7 | gemspec path: "../" 8 | -------------------------------------------------------------------------------- /gemfiles/fluentd_0.14.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "fluentd", ">= 0.14" 6 | 7 | gemspec path: "../" 8 | -------------------------------------------------------------------------------- /lib/fluent/plugin/in_elb_access_log.rb: -------------------------------------------------------------------------------- 1 | require 'csv' 2 | require 'fileutils' 3 | require 'logger' 4 | require 'time' 5 | require 'addressable/uri' 6 | require 'aws-sdk-s3' 7 | require 'multiple_files_gzip_reader' 8 | 9 | require 'fluent/input' 10 | require 'fluent_plugin_elb_access_log/version' 11 | 12 | class FluentPluginElbAccessLogInput < Fluent::Input 13 | Fluent::Plugin.register_input('elb_access_log', self) 14 | 15 | USER_AGENT_SUFFIX = "fluent-plugin-elb-access-log/#{FluentPluginElbAccessLog::VERSION}" 16 | 17 | ACCESS_LOG_FIELDS = { 18 | # http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html 19 | 'clb' => { 20 | 'timestamp' => nil, 21 | 'elb' => nil, 22 | 'client_port' => :to_i, 23 | 'backend_port' => :to_i, 24 | 'request_processing_time' => :to_f, 25 | 'backend_processing_time' => :to_f, 26 | 'response_processing_time' => :to_f, 27 | 'elb_status_code' => :to_i, 28 | 'backend_status_code' => :to_i, 29 | 'received_bytes' => :to_i, 30 | 'sent_bytes' => :to_i, 31 | 'request' => nil, 32 | 'user_agent' => nil, 33 | 'ssl_cipher' => nil, 34 | 'ssl_protocol' => nil, 35 | }, 36 | # http://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html 37 | 'alb' => { 38 | 'type' => nil, 39 | 'timestamp' => nil, 40 | 'elb' => nil, 41 | 'client_port' => :to_i, 42 | 'target_port' => :to_i, 43 | 'request_processing_time' => :to_f, 44 | 'target_processing_time' => :to_f, 45 | 'response_processing_time' => :to_f, 46 | 'elb_status_code' => :to_i, 47 | 'target_status_code' => :to_i, 48 | 'received_bytes' => :to_i, 49 | 'sent_bytes' => :to_i, 50 | 'request' => nil, 51 | 'user_agent' => nil, 52 | 'ssl_cipher' => nil, 53 | 'ssl_protocol' => nil, 54 | 'target_group_arn' => nil, 55 | 'trace_id' => nil, 56 | 'domain_name' => nil, 57 | 'chosen_cert_arn' => nil, 58 | }, 59 | } 60 | 61 | ELB_TYPES = ACCESS_LOG_FIELDS.keys 62 | 63 | config_param :elb_type, :string, default: 'clb' 64 | config_param :aws_key_id, :string, default: nil, secret: true 65 | config_param :aws_sec_key, :string, default: nil, secret: true 66 | config_param :profile, :string, default: nil 67 | config_param :credentials_path, :string, default: nil 68 | config_param :http_proxy, :string, default: nil 69 | config_param :account_id, :string 70 | config_param :region, :string 71 | config_param :s3_bucket, :string 72 | config_param :s3_prefix, :string, default: nil 73 | config_param :tag, :string, default: 'elb.access_log' 74 | config_param :tsfile_path, :string, default: '/var/tmp/fluent-plugin-elb-access-log.ts' 75 | config_param :histfile_path, :string, default: '/var/tmp/fluent-plugin-elb-access-log.history' 76 | config_param :interval, :time, default: 300 77 | config_param :start_datetime, :string, default: nil 78 | config_param :buffer_sec, :time, default: 600 79 | config_param :history_length, :integer, default: 100 80 | config_param :sampling_interval, :integer, default: 1 81 | config_param :debug, :bool, default: false 82 | config_param :filter, :hash, default: nil 83 | config_param :filter_operator, :string, default: 'and' 84 | config_param :type_cast, :bool, default: true 85 | config_param :parse_request, :bool, default: true 86 | config_param :split_addr_port, :bool, default: true 87 | config_param :file_filter, :string, default: nil 88 | config_param :request_separator, :string, default: '.' 89 | 90 | def configure(conf) 91 | super 92 | 93 | unless ELB_TYPES.include?(@elb_type) 94 | raise Fluent::ConfigError, "Invalid ELB type: #{@elb_type}" 95 | end 96 | 97 | unless %w(and or).include?(@filter_operator) 98 | raise Fluent::ConfigError, "Invalid filter operator: #{@filter_operator}" 99 | end 100 | 101 | FileUtils.touch(@tsfile_path) 102 | FileUtils.touch(@histfile_path) 103 | tsfile_start_datetime = parse_tsfile 104 | 105 | if @start_datetime and not tsfile_start_datetime 106 | @start_datetime = Time.parse(@start_datetime).utc 107 | else 108 | if @start_datetime 109 | log.warn("start_datetime(#{@start_datetime}) is set. but tsfile datetime(#{tsfile_start_datetime}) is used") 110 | end 111 | 112 | @start_datetime = tsfile_start_datetime || Time.now.utc 113 | end 114 | 115 | @history = load_history 116 | 117 | if @filter 118 | @filter = Hash[@filter.map {|k, v| [k.to_s, Regexp.new(v.to_s)] }] 119 | end 120 | 121 | if @file_filter 122 | @file_filter = Regexp.new(@file_filter) 123 | end 124 | end 125 | 126 | def start 127 | super 128 | 129 | # Load client 130 | client 131 | 132 | @loop = Coolio::Loop.new 133 | timestamp = @start_datetime 134 | 135 | timer = TimerWatcher.new(@interval, true, log) do 136 | new_timestamp = fetch(timestamp) 137 | 138 | if new_timestamp > timestamp 139 | save_timestamp(new_timestamp) 140 | timestamp = new_timestamp 141 | end 142 | 143 | if @history.length > @history_length 144 | @history.shift(@history.length - @history_length) 145 | end 146 | 147 | save_history 148 | end 149 | 150 | @loop.attach(timer) 151 | @thread = Thread.new(&method(:run)) 152 | end 153 | 154 | def shutdown 155 | @loop.stop 156 | @thread.kill 157 | @thread.join 158 | super 159 | end 160 | 161 | private 162 | 163 | def run 164 | @loop.run 165 | rescue => e 166 | log.error(e.message) 167 | log.error_backtrace(e.backtrace) 168 | end 169 | 170 | def fetch(timestamp) 171 | last_timestamp = timestamp 172 | 173 | prefixes(timestamp).each do |prefix| 174 | client.list_objects_v2(bucket: @s3_bucket, prefix: prefix).each do |page| 175 | page.contents.each do |obj| 176 | if @file_filter and obj.key !~ @file_filter 177 | next 178 | end 179 | 180 | account_id, logfile_const, region, elb_name, logfile_datetime, ip, logfile_suffix = obj.key.split('_', 7) 181 | logfile_datetime = Time.parse(logfile_datetime) 182 | 183 | if logfile_suffix !~ /\.log(\.gz)?\z/ or logfile_datetime <= (timestamp - @buffer_sec) 184 | next 185 | end 186 | 187 | unless @history.include?(obj.key) 188 | access_log = client.get_object(bucket: @s3_bucket, key: obj.key).body 189 | 190 | if obj.key.end_with?('.gz') 191 | begin 192 | access_log = MultipleFilesGzipReader.new(access_log) 193 | 194 | # check gzip format 195 | access_log.first 196 | access_log.rewind 197 | rescue Zlib::Error => e 198 | @log.warn("#{e.message}: #{access_log.inspect.slice(0, 64)}") 199 | next 200 | end 201 | else 202 | access_log = access_log.each_line 203 | end 204 | 205 | emit_access_log(access_log) 206 | last_timestamp = logfile_datetime 207 | @history.push(obj.key) 208 | end 209 | end 210 | end 211 | end 212 | 213 | last_timestamp 214 | end 215 | 216 | def prefixes(timestamp) 217 | base_prefix = "AWSLogs/#{@account_id}/elasticloadbalancing/#{@region}/" 218 | base_prefix = "#{@s3_prefix}/#{base_prefix}" if @s3_prefix 219 | 220 | [timestamp - 86400, timestamp, timestamp + 86400].map do |date| 221 | base_prefix + date.strftime('%Y/%m/%d/') 222 | end 223 | end 224 | 225 | def emit_access_log(access_log) 226 | if @sampling_interval > 1 227 | access_log = sampling(access_log) 228 | end 229 | 230 | records = parse_log(access_log) 231 | 232 | records.each do |record| 233 | begin 234 | time = Time.parse(record['timestamp']) 235 | router.emit(@tag, time.to_i, record) 236 | rescue ArgumentError => e 237 | @log.warn("#{e.message}: #{record}") 238 | @log.warn('A record that has bad timestamp is not emitted.') 239 | end 240 | end 241 | end 242 | 243 | def parse_log(access_log) 244 | parsed_access_log = [] 245 | 246 | access_log.each do |line| 247 | line.chomp! 248 | 249 | case @elb_type 250 | when 'clb' 251 | line = parse_clb_line(line) 252 | when 'alb' 253 | line = parse_alb_line(line) 254 | end 255 | 256 | parsed_access_log << line if line 257 | end 258 | 259 | records = [] 260 | access_log_fields = ACCESS_LOG_FIELDS.fetch(@elb_type) 261 | 262 | parsed_access_log.each do |row| 263 | record = Hash[access_log_fields.keys.zip(row)] 264 | 265 | split_address_port!(record, 'client') 266 | 267 | case @elb_type 268 | when 'clb' 269 | split_address_port!(record, 'backend') 270 | when 'alb' 271 | split_address_port!(record, 'target') 272 | end 273 | 274 | if @filter 275 | if @filter_operator == 'or' 276 | next if @filter.all? {|k, r| record[k] !~ r } 277 | else 278 | next if @filter.any? {|k, r| record[k] !~ r } 279 | end 280 | end 281 | 282 | if @type_cast 283 | access_log_fields.each do |name, conv| 284 | if conv and (value = record[name]) 285 | record[name] = value.send(conv) 286 | end 287 | end 288 | end 289 | 290 | if @parse_request 291 | parse_request!(record) 292 | end 293 | 294 | records << record 295 | end 296 | 297 | records 298 | end 299 | 300 | def parse_clb_line(line) 301 | parsed = nil 302 | 303 | begin 304 | parsed = CSV.parse_line(line, col_sep: ' ') 305 | rescue => e 306 | begin 307 | parsed = line.split(' ', 12) 308 | 309 | # request 310 | parsed[11] ||= '' 311 | parsed[11].sub!(/\A"/, '') 312 | parsed[11].sub!(/"(.*)\z/, '') 313 | 314 | user_agent, ssl_cipher, ssl_protocol = rsplit($1.strip, ' ', 3) 315 | 316 | parsed[12] = unquote(user_agent) 317 | parsed[13] = ssl_cipher 318 | parsed[14] = ssl_protocol 319 | rescue => e2 320 | @log.warn("#{e.message}: #{e2.message}: #{line}") 321 | end 322 | end 323 | 324 | parsed 325 | end 326 | 327 | def parse_alb_line(line) 328 | parsed = nil 329 | 330 | begin 331 | parsed = CSV.parse_line(line, col_sep: ' ') 332 | rescue => e 333 | begin 334 | parsed = line.split(' ', 13) 335 | 336 | # request 337 | parsed[12] ||= '' 338 | parsed[12].sub!(/\A"/, '') 339 | parsed[12].sub!(/"(.*)\z/, '') 340 | 341 | user_agent, ssl_cipher, ssl_protocol, target_group_arn, trace_id, domain_name, chosen_cert_arn = rsplit($1.strip, ' ', 7) 342 | 343 | parsed[13] = unquote(user_agent) 344 | parsed[14] = ssl_cipher 345 | parsed[15] = ssl_protocol 346 | parsed[16] = target_group_arn 347 | parsed[17] = unquote(trace_id) 348 | parsed[18] = unquote(domain_name) 349 | parsed[19] = unquote(chosen_cert_arn) 350 | rescue => e2 351 | @log.warn("#{e.message}: #{e2.message}: #{line}") 352 | end 353 | end 354 | 355 | parsed 356 | end 357 | 358 | def sampling(access_log) 359 | access_log.each_with_index.select {|_, i| (i % @sampling_interval).zero? }.map(&:first) 360 | end 361 | 362 | def split_address_port!(record, prefix) 363 | address_port = record["#{prefix}_port"] 364 | return unless address_port 365 | 366 | if @split_addr_port 367 | address, port = address_port.split(':', 2) 368 | record[prefix] = address 369 | record["#{prefix}_port"] = port 370 | else 371 | record[prefix] = address_port 372 | record.delete("#{prefix}_port") 373 | end 374 | end 375 | 376 | def parse_request!(record) 377 | request = record['request'] 378 | return unless request 379 | method, uri, http_version = request.split(' ', 3) 380 | 381 | record["request#{@request_separator}method"] = method 382 | record["request#{@request_separator}uri"] = uri 383 | record["request#{@request_separator}http_version"] = http_version 384 | 385 | begin 386 | uri = Addressable::URI.parse(uri) 387 | 388 | if uri 389 | [:scheme ,:user, :host, :port, :path, :query, :fragment].each do |key| 390 | value = uri.send(key) 391 | 392 | if not @type_cast and key == :port 393 | value = value.to_s 394 | end 395 | 396 | record["request#{@request_separator}uri#{@request_separator}#{key}"] = value 397 | end 398 | end 399 | rescue => e 400 | @log.warn("#{e.message}: #{uri}") 401 | end 402 | end 403 | 404 | def save_timestamp(timestamp) 405 | open(@tsfile_path, 'w') do |tsfile| 406 | tsfile << timestamp.to_s 407 | end 408 | end 409 | 410 | def load_history 411 | File.read(@histfile_path).split("\n") 412 | end 413 | 414 | def save_history 415 | open(@histfile_path, 'w') do |histfile| 416 | histfile << @history.join("\n") 417 | end 418 | end 419 | 420 | def parse_tsfile 421 | Time.parse(File.read(@tsfile_path)).utc 422 | rescue 423 | nil 424 | end 425 | 426 | def client 427 | return @client if @client 428 | 429 | options = {user_agent_suffix: USER_AGENT_SUFFIX} 430 | options[:region] = @region if @region 431 | options[:http_proxy] = @http_proxy if @http_proxy 432 | 433 | if @aws_key_id and @aws_sec_key 434 | options[:access_key_id] = @aws_key_id 435 | options[:secret_access_key] = @aws_sec_key 436 | elsif @profile 437 | credentials_opts = {profile_name: @profile} 438 | credentials_opts[:path] = @credentials_path if @credentials_path 439 | credentials = Aws::SharedCredentials.new(credentials_opts) 440 | options[:credentials] = credentials 441 | end 442 | 443 | if @debug 444 | options[:logger] = Logger.new(log.out) 445 | options[:log_level] = :debug 446 | #options[:http_wire_trace] = true 447 | end 448 | 449 | @client = Aws::S3::Client.new(options) 450 | end 451 | 452 | def rsplit(str, sep, n) 453 | str = str.dup 454 | substrs = [] 455 | 456 | (n - 1).times do 457 | pos = str.rindex(sep) 458 | next unless pos 459 | substr = str.slice!(pos..-1).slice(sep.length..-1) 460 | substrs << substr 461 | end 462 | 463 | substrs << str 464 | substrs.reverse 465 | end 466 | 467 | def unquote(str) 468 | return nil if (str || '').empty? 469 | str.sub(/\A"/, '').sub(/"\z/, '') 470 | end 471 | 472 | class TimerWatcher < Coolio::TimerWatcher 473 | def initialize(interval, repeat, log, &callback) 474 | @callback = callback 475 | @log = log 476 | super(interval, repeat) 477 | end 478 | 479 | def on_timer 480 | @callback.call 481 | rescue => e 482 | @log.error(e.message) 483 | @log.error_backtrace(e.backtrace) 484 | end 485 | end # TimerWatcher 486 | end # FluentPluginElbAccessLogInput 487 | -------------------------------------------------------------------------------- /lib/fluent_plugin_elb_access_log/version.rb: -------------------------------------------------------------------------------- 1 | module FluentPluginElbAccessLog 2 | VERSION = '0.6.1' 3 | end 4 | -------------------------------------------------------------------------------- /spec/in_elb_access_log_alb_spec.rb: -------------------------------------------------------------------------------- 1 | describe FluentPluginElbAccessLogInput do 2 | let(:account_id) { '123456789012' } 3 | let(:s3_bucket) { 'my-bucket' } 4 | let(:region) { 'us-west-1' } 5 | let(:driver) { create_driver(fluentd_conf) } 6 | let!(:client){ Aws::S3::Client.new(stub_responses: true) } 7 | 8 | let(:fluentd_conf) do 9 | { 10 | interval: 0, 11 | account_id: account_id, 12 | s3_bucket: s3_bucket, 13 | region: region, 14 | start_datetime: (today - 1).to_s, 15 | elb_type: 'alb', 16 | } 17 | end 18 | 19 | let(:today) { Time.parse('2015/05/24 18:30 UTC') } 20 | let(:yesterday) { today - 86400 } 21 | let(:tomorrow) { today + 86400 } 22 | 23 | let(:today_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{today.strftime('%Y/%m/%d')}/" } 24 | let(:yesterday_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{yesterday.strftime('%Y/%m/%d')}/" } 25 | let(:tomorrow_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{tomorrow.strftime('%Y/%m/%d')}/" } 26 | 27 | let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{today.iso8601}_52.68.51.1_8hSqR3o4.log.gz" } 28 | let(:yesterday_object_key) { "#{yesterday_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{yesterday.iso8601}_52.68.51.1_8hSqR3o4.log.gz" } 29 | let(:tomorrow_object_key) { "#{tomorrow_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{tomorrow.iso8601}_52.68.51.1_8hSqR3o4.log.gz" } 30 | 31 | before do 32 | Timecop.freeze(today) 33 | allow(Aws::S3::Client).to receive(:new) { client } 34 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:load_history) { [] } 35 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:parse_tsfile) { nil } 36 | allow(FileUtils).to receive(:touch) 37 | expect(driver.instance.log).to_not receive(:error) 38 | end 39 | 40 | after do 41 | Timecop.return 42 | end 43 | 44 | subject { driver_events } 45 | 46 | context 'when access log does not exist' do 47 | before do 48 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 49 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] } 50 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 51 | expect(driver.instance).to_not receive(:save_timestamp).with(today) 52 | expect(driver.instance.log).to_not receive(:warn) 53 | 54 | driver_run(driver) 55 | end 56 | 57 | it { is_expected.to be_empty } 58 | end 59 | 60 | context 'when access log exists' do 61 | let(:today_access_log) do 62 | gzip(<<-EOS) 63 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 64 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 65 | EOS 66 | end 67 | 68 | let(:tomorrow_access_log) do 69 | gzip(<<-EOS) 70 | https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 71 | https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 72 | EOS 73 | end 74 | 75 | before do 76 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do 77 | [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])] 78 | end 79 | 80 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 81 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 82 | end 83 | 84 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do 85 | [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])] 86 | end 87 | 88 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 89 | double('today_s3_object', body: StringIO.new(today_access_log)) 90 | end 91 | 92 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do 93 | double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log)) 94 | end 95 | 96 | expect(driver.instance).to receive(:save_timestamp).with(tomorrow) 97 | expect(driver.instance.log).to_not receive(:warn) 98 | 99 | driver_run(driver) 100 | end 101 | 102 | let(:expected_emits) do 103 | [["elb.access_log", 104 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 105 | {"type"=>"https", 106 | "timestamp"=>"2015-05-24T19:55:36.000000Z", 107 | "elb"=>"hoge", 108 | "client_port"=>57673, 109 | "target_port"=>80, 110 | "request_processing_time"=>5.3e-05, 111 | "target_processing_time"=>0.000913, 112 | "response_processing_time"=>3.6e-05, 113 | "elb_status_code"=>200, 114 | "target_status_code"=>200, 115 | "received_bytes"=>0, 116 | "sent_bytes"=>3, 117 | "request"=> 118 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 119 | "user_agent"=>"curl/7.30.0", 120 | "ssl_cipher"=>"ssl_cipher", 121 | "ssl_protocol"=>"ssl_protocol", 122 | "target_group_arn"=> 123 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 124 | "trace_id"=>"Root=xxx", 125 | "domain_name"=>"-", 126 | "chosen_cert_arn"=> 127 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 128 | "client"=>"14.14.124.20", 129 | "target"=>"10.0.199.184", 130 | "request.method"=>"GET", 131 | "request.uri"=> 132 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 133 | "request.http_version"=>"HTTP/1.1", 134 | "request.uri.scheme"=>"http", 135 | "request.uri.user"=>nil, 136 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 137 | "request.uri.port"=>80, 138 | "request.uri.path"=>"/", 139 | "request.uri.query"=>nil, 140 | "request.uri.fragment"=>nil}], 141 | ["elb.access_log", 142 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 143 | {"type"=>"https", 144 | "timestamp"=>"2015-05-24T19:55:36.000000Z", 145 | "elb"=>"hoge", 146 | "client_port"=>57673, 147 | "target_port"=>80, 148 | "request_processing_time"=>5.3e-05, 149 | "target_processing_time"=>0.000913, 150 | "response_processing_time"=>3.6e-05, 151 | "elb_status_code"=>200, 152 | "target_status_code"=>200, 153 | "received_bytes"=>0, 154 | "sent_bytes"=>3, 155 | "request"=> 156 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 157 | "user_agent"=>"curl/7.30.0", 158 | "ssl_cipher"=>"ssl_cipher", 159 | "ssl_protocol"=>"ssl_protocol", 160 | "target_group_arn"=> 161 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 162 | "trace_id"=>"Root=xxx", 163 | "domain_name"=>"-", 164 | "chosen_cert_arn"=> 165 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 166 | "client"=>"14.14.124.20", 167 | "target"=>"10.0.199.184", 168 | "request.method"=>"GET", 169 | "request.uri"=> 170 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 171 | "request.http_version"=>"HTTP/1.1", 172 | "request.uri.scheme"=>"http", 173 | "request.uri.user"=>nil, 174 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 175 | "request.uri.port"=>80, 176 | "request.uri.path"=>"/", 177 | "request.uri.query"=>nil, 178 | "request.uri.fragment"=>nil}], 179 | ["elb.access_log", 180 | Time.parse('2015-05-25 19:55:36 UTC').to_i, 181 | {"type"=>"https", 182 | "timestamp"=>"2015-05-25T19:55:36.000000Z", 183 | "elb"=>"hoge", 184 | "client_port"=>57673, 185 | "target_port"=>80, 186 | "request_processing_time"=>5.3e-05, 187 | "target_processing_time"=>0.000913, 188 | "response_processing_time"=>3.6e-05, 189 | "elb_status_code"=>200, 190 | "target_status_code"=>200, 191 | "received_bytes"=>0, 192 | "sent_bytes"=>3, 193 | "request"=> 194 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 195 | "user_agent"=>"curl/7.30.0", 196 | "ssl_cipher"=>"ssl_cipher", 197 | "ssl_protocol"=>"ssl_protocol", 198 | "target_group_arn"=> 199 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 200 | "trace_id"=>"Root=xxx", 201 | "domain_name"=>"-", 202 | "chosen_cert_arn"=> 203 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 204 | "client"=>"14.14.124.20", 205 | "target"=>"10.0.199.184", 206 | "request.method"=>"GET", 207 | "request.uri"=> 208 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 209 | "request.http_version"=>"HTTP/1.1", 210 | "request.uri.scheme"=>"http", 211 | "request.uri.user"=>nil, 212 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 213 | "request.uri.port"=>80, 214 | "request.uri.path"=>"/", 215 | "request.uri.query"=>nil, 216 | "request.uri.fragment"=>nil}], 217 | ["elb.access_log", 218 | Time.parse('2015-05-25 19:55:36 UTC').to_i, 219 | {"type"=>"https", 220 | "timestamp"=>"2015-05-25T19:55:36.000000Z", 221 | "elb"=>"hoge", 222 | "client_port"=>57673, 223 | "target_port"=>80, 224 | "request_processing_time"=>5.3e-05, 225 | "target_processing_time"=>0.000913, 226 | "response_processing_time"=>3.6e-05, 227 | "elb_status_code"=>200, 228 | "target_status_code"=>200, 229 | "received_bytes"=>0, 230 | "sent_bytes"=>3, 231 | "request"=> 232 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 233 | "user_agent"=>"curl/7.30.0", 234 | "ssl_cipher"=>"ssl_cipher", 235 | "ssl_protocol"=>"ssl_protocol", 236 | "target_group_arn"=> 237 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 238 | "trace_id"=>"Root=xxx", 239 | "domain_name"=>"-", 240 | "chosen_cert_arn"=> 241 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 242 | "client"=>"14.14.124.20", 243 | "target"=>"10.0.199.184", 244 | "request.method"=>"GET", 245 | "request.uri"=> 246 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 247 | "request.http_version"=>"HTTP/1.1", 248 | "request.uri.scheme"=>"http", 249 | "request.uri.user"=>nil, 250 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 251 | "request.uri.port"=>80, 252 | "request.uri.path"=>"/", 253 | "request.uri.query"=>nil, 254 | "request.uri.fragment"=>nil}]] 255 | end 256 | 257 | it { is_expected.to match_table expected_emits } 258 | 259 | context 'when sampling' do 260 | let(:fluentd_conf) do 261 | { 262 | interval: 0, 263 | account_id: account_id, 264 | s3_bucket: s3_bucket, 265 | region: region, 266 | start_datetime: (today - 1).to_s, 267 | sampling_interval: 2, 268 | elb_type: 'alb', 269 | } 270 | end 271 | 272 | it do 273 | expected_emits.delete_at(3) 274 | expected_emits.delete_at(1) 275 | is_expected.to match_table expected_emits 276 | end 277 | end 278 | 279 | context 'with filter' do 280 | let(:fluentd_conf) do 281 | { 282 | interval: 0, 283 | account_id: account_id, 284 | s3_bucket: s3_bucket, 285 | region: region, 286 | start_datetime: (today - 1).to_s, 287 | elb_type: 'alb', 288 | filter: '{"timestamp": "2015-05-25"}', 289 | } 290 | end 291 | 292 | it do 293 | expected_emits.slice!(0, 2) 294 | is_expected.to match_table expected_emits 295 | end 296 | end 297 | 298 | context 'with filter (or)' do 299 | let(:fluentd_conf) do 300 | { 301 | interval: 0, 302 | account_id: account_id, 303 | s3_bucket: s3_bucket, 304 | region: region, 305 | start_datetime: (today - 1).to_s, 306 | elb_type: 'alb', 307 | filter: '{"timestamp": "2015-05-25"}', 308 | filter_operator: 'or' 309 | } 310 | end 311 | 312 | it do 313 | expected_emits.slice!(0, 2) 314 | is_expected.to match_table expected_emits 315 | end 316 | end 317 | 318 | context 'with filter (or/multi)' do 319 | let(:fluentd_conf) do 320 | { 321 | interval: 0, 322 | account_id: account_id, 323 | s3_bucket: s3_bucket, 324 | region: region, 325 | start_datetime: (today - 1).to_s, 326 | elb_type: 'alb', 327 | filter: '{"timestamp": "2015-05-25", "elb_status_code": "^2"}', 328 | filter_operator: 'or' 329 | } 330 | end 331 | 332 | it do 333 | is_expected.to match_table expected_emits 334 | end 335 | end 336 | 337 | context 'without type cast' do 338 | let(:fluentd_conf) do 339 | { 340 | interval: 0, 341 | account_id: account_id, 342 | s3_bucket: s3_bucket, 343 | region: region, 344 | start_datetime: (today - 1).to_s, 345 | elb_type: 'alb', 346 | type_cast: 'false', 347 | } 348 | end 349 | 350 | it do 351 | expected_emits_without_type_cast = expected_emits.map do |tag, ts, h| 352 | h = Hash[h.map {|k, v| 353 | v = case v 354 | when nil 355 | v 356 | when Float 357 | "%.6f" % v 358 | else 359 | v.to_s 360 | end 361 | 362 | [k, v] 363 | }] 364 | 365 | [tag, ts, h] 366 | end 367 | 368 | is_expected.to match_table expected_emits_without_type_cast 369 | end 370 | end 371 | 372 | context 'without request parsing' do 373 | let(:fluentd_conf) do 374 | { 375 | interval: 0, 376 | account_id: account_id, 377 | s3_bucket: s3_bucket, 378 | region: region, 379 | start_datetime: (today - 1).to_s, 380 | elb_type: 'alb', 381 | parse_request: 'false', 382 | } 383 | end 384 | 385 | it do 386 | expected_emits_without_request_parsing = expected_emits.map do |tag, ts, h| 387 | h = Hash[h.select {|k, v| k !~ /\Arequest\./ }] 388 | [tag, ts, h] 389 | end 390 | 391 | is_expected.to match_table expected_emits_without_request_parsing 392 | end 393 | end 394 | 395 | context 'without addr/port splitting' do 396 | let(:fluentd_conf) do 397 | { 398 | interval: 0, 399 | account_id: account_id, 400 | s3_bucket: s3_bucket, 401 | region: region, 402 | start_datetime: (today - 1).to_s, 403 | elb_type: 'alb', 404 | split_addr_port: 'false', 405 | } 406 | end 407 | 408 | it do 409 | expected_emits_without_request_parsing = expected_emits.map do |tag, ts, h| 410 | h.keys.select {|k| k =~ /_port\z/ }.each do |prefix_port| 411 | prefix, _ = prefix_port.split('_', 2) 412 | h[prefix] = h[prefix] + ':' + h[prefix_port].to_s 413 | h.delete(prefix_port) 414 | end 415 | 416 | [tag, ts, h] 417 | end 418 | 419 | is_expected.to match_table expected_emits_without_request_parsing 420 | end 421 | end 422 | 423 | context 'with request_separator' do 424 | let(:fluentd_conf) do 425 | { 426 | interval: 0, 427 | account_id: account_id, 428 | s3_bucket: s3_bucket, 429 | region: region, 430 | start_datetime: (today - 1).to_s, 431 | elb_type: 'alb', 432 | request_separator: '_' 433 | } 434 | end 435 | 436 | it do 437 | expected_emits_with_underscore = expected_emits.map do |tag, ts, h| 438 | h = Hash[h.map {|k, v| [k.gsub('.', '_'), v] }] 439 | [tag, ts, h] 440 | end 441 | 442 | is_expected.to match_table expected_emits_with_underscore 443 | end 444 | end 445 | end 446 | 447 | context 'with file_filter' do 448 | let(:today_access_log) do 449 | gzip(<<-EOS) 450 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 451 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 452 | EOS 453 | end 454 | 455 | let(:tomorrow_access_log) do 456 | gzip(<<-EOS) 457 | https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 458 | https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 459 | EOS 460 | end 461 | 462 | before do 463 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do 464 | [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])] 465 | end 466 | 467 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 468 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 469 | end 470 | 471 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do 472 | [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])] 473 | end 474 | 475 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 476 | double('today_s3_object', body: StringIO.new(today_access_log)) 477 | end 478 | 479 | expect(client).to_not receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do 480 | double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log)) 481 | end 482 | 483 | expect(driver.instance).to receive(:save_timestamp).with(today) 484 | expect(driver.instance.log).to_not receive(:warn) 485 | 486 | driver_run(driver) 487 | end 488 | 489 | let(:expected_emits) do 490 | [["elb.access_log", 491 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 492 | {"type"=>"https", 493 | "timestamp"=>"2015-05-24T19:55:36.000000Z", 494 | "elb"=>"hoge", 495 | "client_port"=>57673, 496 | "target_port"=>80, 497 | "request_processing_time"=>5.3e-05, 498 | "target_processing_time"=>0.000913, 499 | "response_processing_time"=>3.6e-05, 500 | "elb_status_code"=>200, 501 | "target_status_code"=>200, 502 | "received_bytes"=>0, 503 | "sent_bytes"=>3, 504 | "request"=> 505 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 506 | "user_agent"=>"curl/7.30.0", 507 | "ssl_cipher"=>"ssl_cipher", 508 | "ssl_protocol"=>"ssl_protocol", 509 | "target_group_arn"=> 510 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 511 | "trace_id"=>"Root=xxx", 512 | "domain_name"=>"-", 513 | "chosen_cert_arn"=> 514 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 515 | "client"=>"14.14.124.20", 516 | "target"=>"10.0.199.184", 517 | "request.method"=>"GET", 518 | "request.uri"=> 519 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 520 | "request.http_version"=>"HTTP/1.1", 521 | "request.uri.scheme"=>"http", 522 | "request.uri.user"=>nil, 523 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 524 | "request.uri.port"=>80, 525 | "request.uri.path"=>"/", 526 | "request.uri.query"=>nil, 527 | "request.uri.fragment"=>nil}], 528 | ["elb.access_log", 529 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 530 | {"type"=>"https", 531 | "timestamp"=>"2015-05-24T19:55:36.000000Z", 532 | "elb"=>"hoge", 533 | "client_port"=>57673, 534 | "target_port"=>80, 535 | "request_processing_time"=>5.3e-05, 536 | "target_processing_time"=>0.000913, 537 | "response_processing_time"=>3.6e-05, 538 | "elb_status_code"=>200, 539 | "target_status_code"=>200, 540 | "received_bytes"=>0, 541 | "sent_bytes"=>3, 542 | "request"=> 543 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 544 | "user_agent"=>"curl/7.30.0", 545 | "ssl_cipher"=>"ssl_cipher", 546 | "ssl_protocol"=>"ssl_protocol", 547 | "target_group_arn"=> 548 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 549 | "trace_id"=>"Root=xxx", 550 | "domain_name"=>"-", 551 | "chosen_cert_arn"=> 552 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 553 | "client"=>"14.14.124.20", 554 | "target"=>"10.0.199.184", 555 | "request.method"=>"GET", 556 | "request.uri"=> 557 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 558 | "request.http_version"=>"HTTP/1.1", 559 | "request.uri.scheme"=>"http", 560 | "request.uri.user"=>nil, 561 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 562 | "request.uri.port"=>80, 563 | "request.uri.path"=>"/", 564 | "request.uri.query"=>nil, 565 | "request.uri.fragment"=>nil}]] 566 | end 567 | 568 | let(:fluentd_conf) do 569 | { 570 | interval: 0, 571 | account_id: account_id, 572 | s3_bucket: s3_bucket, 573 | region: region, 574 | start_datetime: (today - 1).to_s, 575 | elb_type: 'alb', 576 | file_filter: today.iso8601, 577 | } 578 | end 579 | 580 | it { is_expected.to match_table expected_emits } 581 | end 582 | 583 | context 'when include bad URI' do 584 | let(:today_access_log) do 585 | gzip(<<-EOS) 586 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 587 | EOS 588 | end 589 | 590 | before do 591 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 592 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 593 | 594 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 595 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 596 | end 597 | 598 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 599 | double('today_s3_object', body: StringIO.new(today_access_log)) 600 | end 601 | 602 | expect(driver.instance).to receive(:save_timestamp).with(today) 603 | 604 | allow(Addressable::URI).to receive(:parse).and_raise('parse error') 605 | expect(driver.instance.log).to receive(:warn).with('parse error: http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/') 606 | 607 | driver_run(driver) 608 | end 609 | 610 | let(:expected_emits) do 611 | [["elb.access_log", 612 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 613 | {"chosen_cert_arn"=> 614 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 615 | "client"=>"14.14.124.20", 616 | "client_port"=>57673, 617 | "domain_name"=>"-", 618 | "elb"=>"hoge", 619 | "elb_status_code"=>200, 620 | "received_bytes"=>0, 621 | "request"=> 622 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 623 | "request.http_version"=>"HTTP/1.1", 624 | "request.method"=>"GET", 625 | "request.uri"=> 626 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 627 | "request_processing_time"=>5.3e-05, 628 | "response_processing_time"=>3.6e-05, 629 | "sent_bytes"=>3, 630 | "ssl_cipher"=>"ssl_cipher", 631 | "ssl_protocol"=>"ssl_protocol", 632 | "target"=>"10.0.199.184", 633 | "target_group_arn"=> 634 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 635 | "target_port"=>80, 636 | "target_processing_time"=>0.000913, 637 | "target_status_code"=>200, 638 | "timestamp"=>"2015-05-24T19:55:36.000000Z", 639 | "trace_id"=>"Root=xxx", 640 | "type"=>"https", 641 | "user_agent"=>"curl/7.30.0"}]] 642 | end 643 | 644 | it { is_expected.to match_table expected_emits } 645 | end 646 | 647 | context 'when access log exists (with tag option)' do 648 | let(:today_access_log) do 649 | gzip(<<-EOS) 650 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 651 | EOS 652 | end 653 | 654 | let(:fluentd_conf) do 655 | { 656 | interval: 0, 657 | account_id: account_id, 658 | s3_bucket: s3_bucket, 659 | region: region, 660 | start_datetime: (today - 1).to_s, 661 | tag: 'any.tag', 662 | elb_type: 'alb', 663 | } 664 | end 665 | 666 | before do 667 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 668 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 669 | 670 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 671 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 672 | end 673 | 674 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 675 | double('today_s3_object', body: StringIO.new(today_access_log)) 676 | end 677 | 678 | expect(driver.instance).to receive(:save_timestamp).with(today) 679 | expect(driver.instance.log).to_not receive(:warn) 680 | 681 | driver_run(driver) 682 | end 683 | 684 | let(:expected_emits) do 685 | [["any.tag", 686 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 687 | {"chosen_cert_arn"=> 688 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 689 | "client"=>"14.14.124.20", 690 | "client_port"=>57673, 691 | "domain_name"=>"-", 692 | "elb"=>"hoge", 693 | "elb_status_code"=>200, 694 | "received_bytes"=>0, 695 | "request"=> 696 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 697 | "request.http_version"=>"HTTP/1.1", 698 | "request.method"=>"GET", 699 | "request.uri"=> 700 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 701 | "request.uri.fragment"=>nil, 702 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 703 | "request.uri.path"=>"/", 704 | "request.uri.port"=>80, 705 | "request.uri.query"=>nil, 706 | "request.uri.scheme"=>"http", 707 | "request.uri.user"=>nil, 708 | "request_processing_time"=>5.3e-05, 709 | "response_processing_time"=>3.6e-05, 710 | "sent_bytes"=>3, 711 | "ssl_cipher"=>"ssl_cipher", 712 | "ssl_protocol"=>"ssl_protocol", 713 | "target"=>"10.0.199.184", 714 | "target_group_arn"=> 715 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 716 | "target_port"=>80, 717 | "target_processing_time"=>0.000913, 718 | "target_status_code"=>200, 719 | "timestamp"=>"2015-05-24T19:55:36.000000Z", 720 | "trace_id"=>"Root=xxx", 721 | "type"=>"https", 722 | "user_agent"=>"curl/7.30.0"}]] 723 | end 724 | 725 | it { is_expected.to match_table expected_emits } 726 | end 727 | 728 | 729 | context 'when access old log exists' do 730 | let(:today_access_log) do 731 | gzip(<<-EOS) 732 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 733 | EOS 734 | end 735 | 736 | let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log.gz" } 737 | 738 | before do 739 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 740 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 741 | 742 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 743 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 744 | end 745 | 746 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 747 | double('today_s3_object', body: StringIO.new(today_access_log)) 748 | end 749 | 750 | expect(driver.instance).to_not receive(:save_timestamp) 751 | expect(driver.instance.log).to_not receive(:warn) 752 | 753 | driver_run(driver) 754 | end 755 | 756 | let(:expected_emits) do 757 | [["elb.access_log", 758 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 759 | {"chosen_cert_arn"=> 760 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 761 | "client"=>"14.14.124.20", 762 | "client_port"=>57673, 763 | "domain_name"=>"-", 764 | "elb"=>"hoge", 765 | "elb_status_code"=>200, 766 | "received_bytes"=>0, 767 | "request"=> 768 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 769 | "request.http_version"=>"HTTP/1.1", 770 | "request.method"=>"GET", 771 | "request.uri"=> 772 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 773 | "request.uri.fragment"=>nil, 774 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 775 | "request.uri.path"=>"/", 776 | "request.uri.port"=>80, 777 | "request.uri.query"=>nil, 778 | "request.uri.scheme"=>"http", 779 | "request.uri.user"=>nil, 780 | "request_processing_time"=>5.3e-05, 781 | "response_processing_time"=>3.6e-05, 782 | "sent_bytes"=>3, 783 | "ssl_cipher"=>"ssl_cipher", 784 | "ssl_protocol"=>"ssl_protocol", 785 | "target"=>"10.0.199.184", 786 | "target_group_arn"=> 787 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 788 | "target_port"=>80, 789 | "target_processing_time"=>0.000913, 790 | "target_status_code"=>200, 791 | "timestamp"=>"2015-05-24T19:55:36.000000Z", 792 | "trace_id"=>"Root=xxx", 793 | "type"=>"https", 794 | "user_agent"=>"curl/7.30.0"}]] 795 | end 796 | 797 | it { is_expected.to match_table expected_emits } 798 | end 799 | 800 | context 'when parse error' do 801 | let(:today_access_log) do 802 | gzip(<<-EOS) 803 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 804 | EOS 805 | end 806 | 807 | before do 808 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 809 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 810 | 811 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 812 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 813 | end 814 | 815 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 816 | double('today_s3_object', body: StringIO.new(today_access_log)) 817 | end 818 | 819 | expect(driver.instance).to receive(:save_timestamp).with(today) 820 | 821 | expect(CSV).to receive(:parse_line).and_raise('parse error') 822 | expect(driver.instance.log).to_not receive(:warn) 823 | 824 | driver_run(driver) 825 | end 826 | 827 | let(:expected_emits) do 828 | [["elb.access_log", 829 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 830 | {"chosen_cert_arn"=> 831 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 832 | "client"=>"14.14.124.20", 833 | "client_port"=>57673, 834 | "domain_name"=>"-", 835 | "elb"=>"hoge", 836 | "elb_status_code"=>200, 837 | "received_bytes"=>0, 838 | "request"=> 839 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 840 | "request.http_version"=>"HTTP/1.1", 841 | "request.method"=>"GET", 842 | "request.uri"=> 843 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 844 | "request.uri.fragment"=>nil, 845 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 846 | "request.uri.path"=>"/", 847 | "request.uri.port"=>80, 848 | "request.uri.query"=>nil, 849 | "request.uri.scheme"=>"http", 850 | "request.uri.user"=>nil, 851 | "request_processing_time"=>5.3e-05, 852 | "response_processing_time"=>3.6e-05, 853 | "sent_bytes"=>3, 854 | "ssl_cipher"=>"ssl_cipher", 855 | "ssl_protocol"=>"ssl_protocol", 856 | "target"=>"10.0.199.184", 857 | "target_group_arn"=> 858 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx", 859 | "target_port"=>80, 860 | "target_processing_time"=>0.000913, 861 | "target_status_code"=>200, 862 | "timestamp"=>"2015-05-24T19:55:36.000000Z", 863 | "trace_id"=>"Root=xxx", 864 | "type"=>"https", 865 | "user_agent"=>"curl/7.30.0"}]] 866 | end 867 | 868 | it { is_expected.to match_table expected_emits } 869 | 870 | context 'when no user_agent' do 871 | let(:today_access_log) do 872 | gzip(<<-EOS) 873 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" 874 | EOS 875 | end 876 | 877 | before do 878 | expected_emits[0][2]['user_agent'] = nil 879 | expected_emits[0][2]['ssl_cipher'] = nil 880 | expected_emits[0][2]['ssl_protocol'] = nil 881 | expected_emits[0][2]['target_group_arn'] = nil 882 | expected_emits[0][2]['trace_id'] = nil 883 | expected_emits[0][2]['domain_name'] = nil 884 | expected_emits[0][2]['chosen_cert_arn'] = nil 885 | end 886 | 887 | it { is_expected.to match_table expected_emits } 888 | end 889 | end 890 | 891 | context 'when access old log exists (timeout)' do 892 | let(:today_access_log) do 893 | gzip(<<-EOS) 894 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 895 | EOS 896 | end 897 | 898 | let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log.gz" } 899 | 900 | before do 901 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 902 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 903 | 904 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 905 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 906 | end 907 | 908 | expect(client).to_not receive(:get_object) 909 | expect(driver.instance).to_not receive(:save_timestamp) 910 | expect(driver.instance.log).to_not receive(:warn) 911 | 912 | driver_run(driver) 913 | end 914 | 915 | it { is_expected.to be_empty } 916 | end 917 | 918 | context 'when emitted log exists' do 919 | let(:today_access_log) do 920 | gzip(<<-EOS) 921 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 922 | EOS 923 | end 924 | 925 | before do 926 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 927 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 928 | 929 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 930 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 931 | end 932 | 933 | expect(client).to_not receive(:get_object) 934 | 935 | history = driver.instance.instance_variable_get(:@history) 936 | history << today_object_key 937 | expect(driver.instance).to_not receive(:save_timestamp) 938 | expect(driver.instance.log).to_not receive(:warn) 939 | 940 | driver_run(driver) 941 | end 942 | 943 | it { is_expected.to be_empty } 944 | end 945 | 946 | describe 'history#length' do 947 | let(:today_access_log) do 948 | gzip(<<-EOS) 949 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 950 | EOS 951 | end 952 | 953 | let(:history) { driver.instance.instance_variable_get(:@history) } 954 | 955 | before do 956 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 957 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 958 | 959 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 960 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 961 | end 962 | 963 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 964 | double('today_s3_object', body: StringIO.new(today_access_log)) 965 | end 966 | 967 | expect(driver.instance).to receive(:save_timestamp).with(today) 968 | expect(driver.instance.log).to_not receive(:warn) 969 | end 970 | 971 | subject { history.length } 972 | 973 | context 'when history.length <= 100' do 974 | before do 975 | driver_run(driver) 976 | end 977 | 978 | it { is_expected.to eq 1 } 979 | end 980 | 981 | context 'when history.length > 100' do 982 | before do 983 | history.concat (1..100).map(&:to_s) 984 | driver_run(driver) 985 | end 986 | 987 | it { is_expected.to eq 100 } 988 | end 989 | end 990 | 991 | context 'when no user_agent' do 992 | let(:today_access_log) do 993 | gzip(<<-EOS) 994 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 995 | EOS 996 | end 997 | 998 | before do 999 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 1000 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 1001 | 1002 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 1003 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 1004 | end 1005 | 1006 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 1007 | double('today_s3_object', body: StringIO.new(today_access_log)) 1008 | end 1009 | 1010 | expect(driver.instance).to receive(:save_timestamp).with(today) 1011 | expect(driver.instance.log).to_not receive(:warn) 1012 | 1013 | driver_run(driver) 1014 | end 1015 | 1016 | let(:expected_emits) do 1017 | [["elb.access_log", 1018 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 1019 | {"chosen_cert_arn"=>nil, 1020 | "client"=>"14.14.124.20", 1021 | "client_port"=>57673, 1022 | "domain_name"=>nil, 1023 | "elb"=>"hoge", 1024 | "elb_status_code"=>200, 1025 | "received_bytes"=>0, 1026 | "request"=> 1027 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 1028 | "request.http_version"=>"HTTP/1.1", 1029 | "request.method"=>"GET", 1030 | "request.uri"=> 1031 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 1032 | "request.uri.fragment"=>nil, 1033 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 1034 | "request.uri.path"=>"/", 1035 | "request.uri.port"=>80, 1036 | "request.uri.query"=>nil, 1037 | "request.uri.scheme"=>"http", 1038 | "request.uri.user"=>nil, 1039 | "request_processing_time"=>5.3e-05, 1040 | "response_processing_time"=>3.6e-05, 1041 | "sent_bytes"=>3, 1042 | "ssl_cipher"=>"Root=xxx", 1043 | "ssl_protocol"=>"-", 1044 | "target"=>"10.0.199.184", 1045 | "target_group_arn"=> 1046 | "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx", 1047 | "target_port"=>80, 1048 | "target_processing_time"=>0.000913, 1049 | "target_status_code"=>200, 1050 | "timestamp"=>"2015-05-24T19:55:36.000000Z", 1051 | "trace_id"=>nil, 1052 | "type"=>"https", 1053 | "user_agent"=> 1054 | "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx"}]] 1055 | end 1056 | 1057 | it { is_expected.to match_table expected_emits } 1058 | end 1059 | 1060 | context 'when inflate fails' do 1061 | let(:today_access_log) do 1062 | <<-EOS 1063 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 1064 | EOS 1065 | end 1066 | 1067 | before do 1068 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 1069 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 1070 | 1071 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 1072 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 1073 | end 1074 | 1075 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 1076 | double('today_s3_object', body: StringIO.new(today_access_log)) 1077 | end 1078 | 1079 | end 1080 | 1081 | specify do 1082 | expect(driver.instance.log).to receive(:warn).with(/not in gzip format: /) 1083 | driver_run(driver) 1084 | end 1085 | end 1086 | 1087 | context 'when bad timestamp' do 1088 | let(:today_access_log) do 1089 | gzip(<<-EOS) 1090 | https xxx hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 1091 | EOS 1092 | end 1093 | 1094 | before do 1095 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 1096 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 1097 | 1098 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 1099 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 1100 | end 1101 | 1102 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 1103 | double('today_s3_object', body: StringIO.new(today_access_log)) 1104 | end 1105 | 1106 | expect(driver.instance).to receive(:save_timestamp).with(today) 1107 | end 1108 | 1109 | specify do 1110 | expect(driver.instance.log).to receive(:warn).with(/no time information in "xxx":/) 1111 | expect(driver.instance.log).to receive(:warn).with('A record that has bad timestamp is not emitted.') 1112 | driver_run(driver) 1113 | end 1114 | end 1115 | 1116 | context 'when unquote fails' do 1117 | let(:today_access_log) do 1118 | gzip(<<-EOS) 1119 | https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx" 1120 | EOS 1121 | end 1122 | 1123 | before do 1124 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 1125 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 1126 | 1127 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 1128 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 1129 | end 1130 | 1131 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 1132 | double('today_s3_object', body: StringIO.new(today_access_log)) 1133 | end 1134 | 1135 | expect(driver.instance).to receive(:save_timestamp).with(today) 1136 | 1137 | expect(CSV).to receive(:parse_line).and_raise('parse error') 1138 | expect(driver.instance).to receive(:unquote).and_raise('unquote error') 1139 | end 1140 | 1141 | specify do 1142 | expect(driver.instance.log).to receive(:warn).with(/parse error: unquote error:/) 1143 | driver_run(driver) 1144 | end 1145 | end 1146 | end 1147 | -------------------------------------------------------------------------------- /spec/in_elb_access_log_clb_spec.rb: -------------------------------------------------------------------------------- 1 | describe FluentPluginElbAccessLogInput do 2 | let(:account_id) { '123456789012' } 3 | let(:s3_bucket) { 'my-bucket' } 4 | let(:region) { 'us-west-1' } 5 | let(:driver) { create_driver(fluentd_conf) } 6 | let!(:client){ Aws::S3::Client.new(stub_responses: true) } 7 | 8 | let(:fluentd_conf) do 9 | { 10 | interval: 0, 11 | account_id: account_id, 12 | s3_bucket: s3_bucket, 13 | region: region, 14 | start_datetime: (today - 1).to_s, 15 | } 16 | end 17 | 18 | let(:today) { Time.parse('2015/05/24 18:30 UTC') } 19 | let(:yesterday) { today - 86400 } 20 | let(:tomorrow) { today + 86400 } 21 | 22 | let(:today_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{today.strftime('%Y/%m/%d')}/" } 23 | let(:yesterday_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{yesterday.strftime('%Y/%m/%d')}/" } 24 | let(:tomorrow_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{tomorrow.strftime('%Y/%m/%d')}/" } 25 | 26 | let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{today.iso8601}_52.68.51.1_8hSqR3o4.log" } 27 | let(:yesterday_object_key) { "#{yesterday_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{yesterday.iso8601}_52.68.51.1_8hSqR3o4.log" } 28 | let(:tomorrow_object_key) { "#{tomorrow_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{tomorrow.iso8601}_52.68.51.1_8hSqR3o4.log" } 29 | 30 | before do 31 | Timecop.freeze(today) 32 | allow(Aws::S3::Client).to receive(:new) { client } 33 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:load_history) { [] } 34 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:parse_tsfile) { nil } 35 | allow(FileUtils).to receive(:touch) 36 | expect(driver.instance.log).to_not receive(:error) 37 | end 38 | 39 | after do 40 | Timecop.return 41 | end 42 | 43 | subject { driver_events } 44 | 45 | context 'when access log does not exist' do 46 | before do 47 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 48 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] } 49 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 50 | expect(driver.instance).to_not receive(:save_timestamp).with(today) 51 | expect(driver.instance).to receive(:save_history) 52 | expect(driver.instance.log).to_not receive(:warn) 53 | 54 | driver_run(driver) 55 | end 56 | 57 | it { is_expected.to be_empty } 58 | end 59 | 60 | context 'when access log exists' do 61 | let(:today_access_log) do 62 | <<-EOS 63 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 64 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 65 | EOS 66 | end 67 | 68 | let(:tomorrow_access_log) do 69 | <<-EOS 70 | 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 71 | 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 72 | EOS 73 | end 74 | 75 | before do 76 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do 77 | [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])] 78 | end 79 | 80 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 81 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 82 | end 83 | 84 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do 85 | [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])] 86 | end 87 | 88 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 89 | double('today_s3_object', body: StringIO.new(today_access_log)) 90 | end 91 | 92 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do 93 | double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log)) 94 | end 95 | 96 | expect(driver.instance).to receive(:save_timestamp).with(tomorrow) 97 | expect(driver.instance).to receive(:save_history) 98 | expect(driver.instance.log).to_not receive(:warn) 99 | 100 | driver_run(driver) 101 | end 102 | 103 | let(:expected_emits) do 104 | [["elb.access_log", 105 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 106 | {"timestamp"=>"2015-05-24T19:55:36.000000Z", 107 | "elb"=>"hoge", 108 | "client"=>"14.14.124.20", 109 | "client_port"=>57673, 110 | "backend"=>"10.0.199.184", 111 | "backend_port"=>80, 112 | "request_processing_time"=>5.3e-05, 113 | "backend_processing_time"=>0.000913, 114 | "response_processing_time"=>3.6e-05, 115 | "elb_status_code"=>200, 116 | "backend_status_code"=>200, 117 | "received_bytes"=>0, 118 | "sent_bytes"=>3, 119 | "request"=> 120 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 121 | "user_agent"=>"curl/7.30.0", 122 | "ssl_cipher"=>"ssl_cipher", 123 | "ssl_protocol"=>"ssl_protocol", 124 | "request.method"=>"GET", 125 | "request.uri"=> 126 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 127 | "request.http_version"=>"HTTP/1.1", 128 | "request.uri.scheme"=>"http", 129 | "request.uri.user"=>nil, 130 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 131 | "request.uri.port"=>80, 132 | "request.uri.path"=>"/", 133 | "request.uri.query"=>nil, 134 | "request.uri.fragment"=>nil}], 135 | ["elb.access_log", 136 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 137 | {"timestamp"=>"2015-05-24T19:55:36.000000Z", 138 | "elb"=>"hoge", 139 | "client"=>"14.14.124.20", 140 | "client_port"=>57673, 141 | "backend"=>"10.0.199.184", 142 | "backend_port"=>80, 143 | "request_processing_time"=>5.3e-05, 144 | "backend_processing_time"=>0.000913, 145 | "response_processing_time"=>3.6e-05, 146 | "elb_status_code"=>200, 147 | "backend_status_code"=>200, 148 | "received_bytes"=>0, 149 | "sent_bytes"=>3, 150 | "request"=> 151 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 152 | "user_agent"=>"curl/7.30.0", 153 | "ssl_cipher"=>"ssl_cipher", 154 | "ssl_protocol"=>"ssl_protocol", 155 | "request.method"=>"GET", 156 | "request.uri"=> 157 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 158 | "request.http_version"=>"HTTP/1.1", 159 | "request.uri.scheme"=>"http", 160 | "request.uri.user"=>nil, 161 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 162 | "request.uri.port"=>80, 163 | "request.uri.path"=>"/", 164 | "request.uri.query"=>nil, 165 | "request.uri.fragment"=>nil}], 166 | ["elb.access_log", 167 | Time.parse('2015-05-25 19:55:36 UTC').to_i, 168 | {"timestamp"=>"2015-05-25T19:55:36.000000Z", 169 | "elb"=>"hoge", 170 | "client"=>"14.14.124.20", 171 | "client_port"=>57673, 172 | "backend"=>"10.0.199.184", 173 | "backend_port"=>80, 174 | "request_processing_time"=>5.3e-05, 175 | "backend_processing_time"=>0.000913, 176 | "response_processing_time"=>3.6e-05, 177 | "elb_status_code"=>200, 178 | "backend_status_code"=>200, 179 | "received_bytes"=>0, 180 | "sent_bytes"=>3, 181 | "request"=> 182 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 183 | "user_agent"=>"curl/7.30.0", 184 | "ssl_cipher"=>"ssl_cipher", 185 | "ssl_protocol"=>"ssl_protocol", 186 | "request.method"=>"GET", 187 | "request.uri"=> 188 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 189 | "request.http_version"=>"HTTP/1.1", 190 | "request.uri.scheme"=>"http", 191 | "request.uri.user"=>nil, 192 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 193 | "request.uri.port"=>80, 194 | "request.uri.path"=>"/", 195 | "request.uri.query"=>nil, 196 | "request.uri.fragment"=>nil}], 197 | ["elb.access_log", 198 | Time.parse('2015-05-25 19:55:36 UTC').to_i, 199 | {"timestamp"=>"2015-05-25T19:55:36.000000Z", 200 | "elb"=>"hoge", 201 | "client"=>"14.14.124.20", 202 | "client_port"=>57673, 203 | "backend"=>"10.0.199.184", 204 | "backend_port"=>80, 205 | "request_processing_time"=>5.3e-05, 206 | "backend_processing_time"=>0.000913, 207 | "response_processing_time"=>3.6e-05, 208 | "elb_status_code"=>200, 209 | "backend_status_code"=>200, 210 | "received_bytes"=>0, 211 | "sent_bytes"=>3, 212 | "request"=> 213 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 214 | "user_agent"=>"curl/7.30.0", 215 | "ssl_cipher"=>"ssl_cipher", 216 | "ssl_protocol"=>"ssl_protocol", 217 | "request.method"=>"GET", 218 | "request.uri"=> 219 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 220 | "request.http_version"=>"HTTP/1.1", 221 | "request.uri.scheme"=>"http", 222 | "request.uri.user"=>nil, 223 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 224 | "request.uri.port"=>80, 225 | "request.uri.path"=>"/", 226 | "request.uri.query"=>nil, 227 | "request.uri.fragment"=>nil}]] 228 | end 229 | 230 | it { is_expected.to match_table expected_emits } 231 | 232 | context 'when sampling' do 233 | let(:fluentd_conf) do 234 | { 235 | interval: 0, 236 | account_id: account_id, 237 | s3_bucket: s3_bucket, 238 | region: region, 239 | start_datetime: (today - 1).to_s, 240 | sampling_interval: 2, 241 | } 242 | end 243 | 244 | it do 245 | expected_emits.delete_at(3) 246 | expected_emits.delete_at(1) 247 | is_expected.to match_table expected_emits 248 | end 249 | end 250 | 251 | context 'with filter' do 252 | let(:fluentd_conf) do 253 | { 254 | interval: 0, 255 | account_id: account_id, 256 | s3_bucket: s3_bucket, 257 | region: region, 258 | start_datetime: (today - 1).to_s, 259 | filter: '{"timestamp": "2015-05-25"}', 260 | } 261 | end 262 | 263 | it do 264 | expected_emits.slice!(0, 2) 265 | is_expected.to match_table expected_emits 266 | end 267 | end 268 | 269 | context 'with filter (or)' do 270 | let(:fluentd_conf) do 271 | { 272 | interval: 0, 273 | account_id: account_id, 274 | s3_bucket: s3_bucket, 275 | region: region, 276 | start_datetime: (today - 1).to_s, 277 | filter: '{"timestamp": "2015-05-25"}', 278 | filter_operator: 'or', 279 | } 280 | end 281 | 282 | it do 283 | expected_emits.slice!(0, 2) 284 | is_expected.to match_table expected_emits 285 | end 286 | end 287 | 288 | context 'with filter (or/multi)' do 289 | let(:fluentd_conf) do 290 | { 291 | interval: 0, 292 | account_id: account_id, 293 | s3_bucket: s3_bucket, 294 | region: region, 295 | start_datetime: (today - 1).to_s, 296 | filter: '{"timestamp": "2015-05-25", "elb_status_code": "^2"}', 297 | filter_operator: 'or', 298 | } 299 | end 300 | 301 | it do 302 | is_expected.to match_table expected_emits 303 | end 304 | end 305 | 306 | context 'without type cast' do 307 | let(:fluentd_conf) do 308 | { 309 | interval: 0, 310 | account_id: account_id, 311 | s3_bucket: s3_bucket, 312 | region: region, 313 | start_datetime: (today - 1).to_s, 314 | type_cast: 'false', 315 | } 316 | end 317 | 318 | it do 319 | expected_emits_without_type_cast = expected_emits.map do |tag, ts, h| 320 | h = Hash[h.map {|k, v| 321 | v = case v 322 | when nil 323 | v 324 | when Float 325 | "%.6f" % v 326 | else 327 | v.to_s 328 | end 329 | 330 | [k, v] 331 | }] 332 | 333 | [tag, ts, h] 334 | end 335 | 336 | is_expected.to match_table expected_emits_without_type_cast 337 | end 338 | end 339 | 340 | context 'without request parsing' do 341 | let(:fluentd_conf) do 342 | { 343 | interval: 0, 344 | account_id: account_id, 345 | s3_bucket: s3_bucket, 346 | region: region, 347 | start_datetime: (today - 1).to_s, 348 | parse_request: 'false', 349 | } 350 | end 351 | 352 | it do 353 | expected_emits_without_request_parsing = expected_emits.map do |tag, ts, h| 354 | h = Hash[h.select {|k, v| k !~ /\Arequest\./ }] 355 | [tag, ts, h] 356 | end 357 | 358 | is_expected.to match_table expected_emits_without_request_parsing 359 | end 360 | end 361 | 362 | context 'without addr/port splitting' do 363 | let(:fluentd_conf) do 364 | { 365 | interval: 0, 366 | account_id: account_id, 367 | s3_bucket: s3_bucket, 368 | region: region, 369 | start_datetime: (today - 1).to_s, 370 | split_addr_port: 'false', 371 | } 372 | end 373 | 374 | it do 375 | expected_emits_without_request_parsing = expected_emits.map do |tag, ts, h| 376 | h.keys.select {|k| k =~ /_port\z/ }.each do |prefix_port| 377 | prefix, _ = prefix_port.split('_', 2) 378 | h[prefix] = h[prefix] + ':' + h[prefix_port].to_s 379 | h.delete(prefix_port) 380 | end 381 | 382 | [tag, ts, h] 383 | end 384 | 385 | is_expected.to match_table expected_emits_without_request_parsing 386 | end 387 | end 388 | 389 | context 'with request_separator' do 390 | let(:fluentd_conf) do 391 | { 392 | interval: 0, 393 | account_id: account_id, 394 | s3_bucket: s3_bucket, 395 | region: region, 396 | start_datetime: (today - 1).to_s, 397 | request_separator: '_' 398 | } 399 | end 400 | 401 | it do 402 | expected_emits_with_underscore = expected_emits.map do |tag, ts, h| 403 | h = Hash[h.map {|k, v| [k.gsub('.', '_'), v] }] 404 | [tag, ts, h] 405 | end 406 | 407 | is_expected.to match_table expected_emits_with_underscore 408 | end 409 | end 410 | end 411 | 412 | context 'with file_filter' do 413 | let(:today_access_log) do 414 | <<-EOS 415 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 416 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 417 | EOS 418 | end 419 | 420 | let(:tomorrow_access_log) do 421 | <<-EOS 422 | 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 423 | 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 424 | EOS 425 | end 426 | 427 | before do 428 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do 429 | [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])] 430 | end 431 | 432 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 433 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 434 | end 435 | 436 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do 437 | [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])] 438 | end 439 | 440 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 441 | double('today_s3_object', body: StringIO.new(today_access_log)) 442 | end 443 | 444 | expect(client).to_not receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do 445 | double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log)) 446 | end 447 | 448 | expect(driver.instance).to receive(:save_timestamp).with(today) 449 | expect(driver.instance).to receive(:save_history) 450 | expect(driver.instance.log).to_not receive(:warn) 451 | 452 | driver_run(driver) 453 | end 454 | 455 | let(:expected_emits) do 456 | [["elb.access_log", 457 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 458 | {"timestamp"=>"2015-05-24T19:55:36.000000Z", 459 | "elb"=>"hoge", 460 | "client"=>"14.14.124.20", 461 | "client_port"=>57673, 462 | "backend"=>"10.0.199.184", 463 | "backend_port"=>80, 464 | "request_processing_time"=>5.3e-05, 465 | "backend_processing_time"=>0.000913, 466 | "response_processing_time"=>3.6e-05, 467 | "elb_status_code"=>200, 468 | "backend_status_code"=>200, 469 | "received_bytes"=>0, 470 | "sent_bytes"=>3, 471 | "request"=> 472 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 473 | "user_agent"=>"curl/7.30.0", 474 | "ssl_cipher"=>"ssl_cipher", 475 | "ssl_protocol"=>"ssl_protocol", 476 | "request.method"=>"GET", 477 | "request.uri"=> 478 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 479 | "request.http_version"=>"HTTP/1.1", 480 | "request.uri.scheme"=>"http", 481 | "request.uri.user"=>nil, 482 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 483 | "request.uri.port"=>80, 484 | "request.uri.path"=>"/", 485 | "request.uri.query"=>nil, 486 | "request.uri.fragment"=>nil}], 487 | ["elb.access_log", 488 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 489 | {"timestamp"=>"2015-05-24T19:55:36.000000Z", 490 | "elb"=>"hoge", 491 | "client"=>"14.14.124.20", 492 | "client_port"=>57673, 493 | "backend"=>"10.0.199.184", 494 | "backend_port"=>80, 495 | "request_processing_time"=>5.3e-05, 496 | "backend_processing_time"=>0.000913, 497 | "response_processing_time"=>3.6e-05, 498 | "elb_status_code"=>200, 499 | "backend_status_code"=>200, 500 | "received_bytes"=>0, 501 | "sent_bytes"=>3, 502 | "request"=> 503 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 504 | "user_agent"=>"curl/7.30.0", 505 | "ssl_cipher"=>"ssl_cipher", 506 | "ssl_protocol"=>"ssl_protocol", 507 | "request.method"=>"GET", 508 | "request.uri"=> 509 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 510 | "request.http_version"=>"HTTP/1.1", 511 | "request.uri.scheme"=>"http", 512 | "request.uri.user"=>nil, 513 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 514 | "request.uri.port"=>80, 515 | "request.uri.path"=>"/", 516 | "request.uri.query"=>nil, 517 | "request.uri.fragment"=>nil}]] 518 | end 519 | 520 | let(:fluentd_conf) do 521 | { 522 | interval: 0, 523 | account_id: account_id, 524 | s3_bucket: s3_bucket, 525 | region: region, 526 | start_datetime: (today - 1).to_s, 527 | file_filter: today.iso8601, 528 | } 529 | end 530 | 531 | it { is_expected.to match_table expected_emits } 532 | end 533 | 534 | context 'when include bad URI' do 535 | let(:today_access_log) do 536 | <<-EOS 537 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 538 | EOS 539 | end 540 | 541 | before do 542 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 543 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 544 | 545 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 546 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 547 | end 548 | 549 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 550 | double('today_s3_object', body: StringIO.new(today_access_log)) 551 | end 552 | 553 | expect(driver.instance).to receive(:save_timestamp).with(today) 554 | expect(driver.instance).to receive(:save_history) 555 | 556 | allow(Addressable::URI).to receive(:parse).and_raise('parse error') 557 | expect(driver.instance.log).to receive(:warn).with('parse error: http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/') 558 | 559 | driver_run(driver) 560 | end 561 | 562 | let(:expected_emits) do 563 | [["elb.access_log", 564 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 565 | {"timestamp"=>"2015-05-24T19:55:36.000000Z", 566 | "elb"=>"hoge", 567 | "client"=>"14.14.124.20", 568 | "client_port"=>57673, 569 | "backend"=>"10.0.199.184", 570 | "backend_port"=>80, 571 | "request_processing_time"=>5.3e-05, 572 | "backend_processing_time"=>0.000913, 573 | "response_processing_time"=>3.6e-05, 574 | "elb_status_code"=>200, 575 | "backend_status_code"=>200, 576 | "received_bytes"=>0, 577 | "sent_bytes"=>3, 578 | "request"=> 579 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 580 | "user_agent"=>"curl/7.30.0", 581 | "ssl_cipher"=>"ssl_cipher", 582 | "ssl_protocol"=>"ssl_protocol", 583 | "request.method"=>"GET", 584 | "request.uri"=> 585 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 586 | "request.http_version"=>"HTTP/1.1"}]] 587 | end 588 | 589 | it { is_expected.to match_table expected_emits } 590 | end 591 | 592 | context 'when access log exists (with tag option)' do 593 | let(:today_access_log) do 594 | <<-EOS 595 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 596 | EOS 597 | end 598 | 599 | let(:fluentd_conf) do 600 | { 601 | interval: 0, 602 | account_id: account_id, 603 | s3_bucket: s3_bucket, 604 | region: region, 605 | start_datetime: (today - 1).to_s, 606 | tag: 'any.tag' 607 | } 608 | end 609 | 610 | before do 611 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 612 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 613 | 614 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 615 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 616 | end 617 | 618 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 619 | double('today_s3_object', body: StringIO.new(today_access_log)) 620 | end 621 | 622 | expect(driver.instance).to receive(:save_timestamp).with(today) 623 | expect(driver.instance).to receive(:save_history) 624 | expect(driver.instance.log).to_not receive(:warn) 625 | 626 | driver_run(driver) 627 | end 628 | 629 | let(:expected_emits) do 630 | [["any.tag", 631 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 632 | {"timestamp"=>"2015-05-24T19:55:36.000000Z", 633 | "elb"=>"hoge", 634 | "client"=>"14.14.124.20", 635 | "client_port"=>57673, 636 | "backend"=>"10.0.199.184", 637 | "backend_port"=>80, 638 | "request_processing_time"=>5.3e-05, 639 | "backend_processing_time"=>0.000913, 640 | "response_processing_time"=>3.6e-05, 641 | "elb_status_code"=>200, 642 | "backend_status_code"=>200, 643 | "received_bytes"=>0, 644 | "sent_bytes"=>3, 645 | "request"=> 646 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 647 | "user_agent"=>"curl/7.30.0", 648 | "ssl_cipher"=>"ssl_cipher", 649 | "ssl_protocol"=>"ssl_protocol", 650 | "request.method"=>"GET", 651 | "request.uri"=> 652 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 653 | "request.http_version"=>"HTTP/1.1", 654 | "request.uri.scheme"=>"http", 655 | "request.uri.user"=>nil, 656 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 657 | "request.uri.port"=>80, 658 | "request.uri.path"=>"/", 659 | "request.uri.query"=>nil, 660 | "request.uri.fragment"=>nil}]] 661 | end 662 | 663 | it { is_expected.to match_table expected_emits } 664 | end 665 | 666 | context 'when access old log exists' do 667 | let(:today_access_log) do 668 | <<-EOS 669 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 670 | EOS 671 | end 672 | 673 | let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log" } 674 | 675 | before do 676 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 677 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 678 | 679 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 680 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 681 | end 682 | 683 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 684 | double('today_s3_object', body: StringIO.new(today_access_log)) 685 | end 686 | 687 | expect(driver.instance).to_not receive(:save_timestamp) 688 | expect(driver.instance).to receive(:save_history) 689 | expect(driver.instance.log).to_not receive(:warn) 690 | 691 | driver_run(driver) 692 | end 693 | 694 | let(:expected_emits) do 695 | [["elb.access_log", 696 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 697 | {"timestamp"=>"2015-05-24T19:55:36.000000Z", 698 | "elb"=>"hoge", 699 | "client"=>"14.14.124.20", 700 | "client_port"=>57673, 701 | "backend"=>"10.0.199.184", 702 | "backend_port"=>80, 703 | "request_processing_time"=>5.3e-05, 704 | "backend_processing_time"=>0.000913, 705 | "response_processing_time"=>3.6e-05, 706 | "elb_status_code"=>200, 707 | "backend_status_code"=>200, 708 | "received_bytes"=>0, 709 | "sent_bytes"=>3, 710 | "request"=> 711 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 712 | "user_agent"=>"curl/7.30.0", 713 | "ssl_cipher"=>"ssl_cipher", 714 | "ssl_protocol"=>"ssl_protocol", 715 | "request.method"=>"GET", 716 | "request.uri"=> 717 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 718 | "request.http_version"=>"HTTP/1.1", 719 | "request.uri.scheme"=>"http", 720 | "request.uri.user"=>nil, 721 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 722 | "request.uri.port"=>80, 723 | "request.uri.path"=>"/", 724 | "request.uri.query"=>nil, 725 | "request.uri.fragment"=>nil}]] 726 | end 727 | 728 | it { is_expected.to match_table expected_emits } 729 | end 730 | 731 | context 'when parse error' do 732 | let(:today_access_log) do 733 | <<-EOS 734 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 735 | EOS 736 | end 737 | 738 | before do 739 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 740 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 741 | 742 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 743 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 744 | end 745 | 746 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 747 | double('today_s3_object', body: StringIO.new(today_access_log)) 748 | end 749 | 750 | expect(driver.instance).to receive(:save_timestamp).with(today) 751 | expect(driver.instance).to receive(:save_history) 752 | 753 | expect(CSV).to receive(:parse_line).and_raise('parse error') 754 | expect(driver.instance.log).to_not receive(:warn) 755 | 756 | driver_run(driver) 757 | end 758 | 759 | let(:expected_emits) do 760 | [["elb.access_log", 761 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 762 | {"timestamp"=>"2015-05-24T19:55:36.000000Z", 763 | "elb"=>"hoge", 764 | "client"=>"14.14.124.20", 765 | "client_port"=>57673, 766 | "backend"=>"10.0.199.184", 767 | "backend_port"=>80, 768 | "request_processing_time"=>5.3e-05, 769 | "backend_processing_time"=>0.000913, 770 | "response_processing_time"=>3.6e-05, 771 | "elb_status_code"=>200, 772 | "backend_status_code"=>200, 773 | "received_bytes"=>0, 774 | "sent_bytes"=>3, 775 | "request"=> 776 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 777 | "user_agent"=>"curl/7.30.0", 778 | "ssl_cipher"=>"ssl_cipher", 779 | "ssl_protocol"=>"ssl_protocol", 780 | "request.method"=>"GET", 781 | "request.uri"=> 782 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 783 | "request.http_version"=>"HTTP/1.1", 784 | "request.uri.scheme"=>"http", 785 | "request.uri.user"=>nil, 786 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 787 | "request.uri.port"=>80, 788 | "request.uri.path"=>"/", 789 | "request.uri.query"=>nil, 790 | "request.uri.fragment"=>nil}]] 791 | end 792 | 793 | it { is_expected.to match_table expected_emits } 794 | 795 | context 'when no user_agent' do 796 | let(:today_access_log) do 797 | <<-EOS 798 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" 799 | EOS 800 | end 801 | 802 | before do 803 | expected_emits[0][2]['user_agent'] = nil 804 | expected_emits[0][2]['ssl_cipher'] = nil 805 | expected_emits[0][2]['ssl_protocol'] = nil 806 | end 807 | 808 | it { is_expected.to match_table expected_emits } 809 | end 810 | end 811 | 812 | context 'when access old log exists (timeout)' do 813 | let(:today_access_log) do 814 | <<-EOS 815 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 816 | EOS 817 | end 818 | 819 | let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log" } 820 | 821 | before do 822 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 823 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 824 | 825 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 826 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 827 | end 828 | 829 | expect(client).to_not receive(:get_object) 830 | expect(driver.instance).to_not receive(:save_timestamp) 831 | expect(driver.instance).to receive(:save_history) 832 | expect(driver.instance.log).to_not receive(:warn) 833 | 834 | driver_run(driver) 835 | end 836 | 837 | it { is_expected.to be_empty } 838 | end 839 | 840 | context 'when emitted log exists' do 841 | let(:today_access_log) do 842 | <<-EOS 843 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 844 | EOS 845 | end 846 | 847 | before do 848 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 849 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 850 | 851 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 852 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 853 | end 854 | 855 | expect(client).to_not receive(:get_object) 856 | 857 | history = driver.instance.instance_variable_get(:@history) 858 | history << today_object_key 859 | expect(driver.instance).to_not receive(:save_timestamp) 860 | expect(driver.instance).to receive(:save_history) 861 | expect(driver.instance.log).to_not receive(:warn) 862 | 863 | driver_run(driver) 864 | end 865 | 866 | it { is_expected.to be_empty } 867 | end 868 | 869 | describe 'history#length' do 870 | let(:today_access_log) do 871 | <<-EOS 872 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 873 | EOS 874 | end 875 | 876 | let(:history) { driver.instance.instance_variable_get(:@history) } 877 | 878 | before do 879 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 880 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 881 | 882 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 883 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 884 | end 885 | 886 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 887 | double('today_s3_object', body: StringIO.new(today_access_log)) 888 | end 889 | 890 | expect(driver.instance).to receive(:save_timestamp).with(today) 891 | expect(driver.instance).to receive(:save_history) 892 | expect(driver.instance.log).to_not receive(:warn) 893 | end 894 | 895 | subject { history.length } 896 | 897 | context 'when history.length <= 100' do 898 | before do 899 | driver_run(driver) 900 | end 901 | 902 | it { is_expected.to eq 1 } 903 | end 904 | 905 | context 'when history.length > 100' do 906 | before do 907 | history.concat (1..100).map(&:to_s) 908 | driver_run(driver) 909 | end 910 | 911 | it { is_expected.to eq 100 } 912 | end 913 | end 914 | 915 | context 'when no user_agent' do 916 | let(:today_access_log) do 917 | <<-EOS 918 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" 919 | EOS 920 | end 921 | 922 | before do 923 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 924 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 925 | 926 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 927 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 928 | end 929 | 930 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 931 | double('today_s3_object', body: StringIO.new(today_access_log)) 932 | end 933 | 934 | expect(driver.instance).to receive(:save_timestamp).with(today) 935 | expect(driver.instance).to receive(:save_history) 936 | expect(driver.instance.log).to_not receive(:warn) 937 | 938 | driver_run(driver) 939 | end 940 | 941 | let(:expected_emits) do 942 | [["elb.access_log", 943 | Time.parse('2015-05-24 19:55:36 UTC').to_i, 944 | {"timestamp"=>"2015-05-24T19:55:36.000000Z", 945 | "elb"=>"hoge", 946 | "client"=>"14.14.124.20", 947 | "client_port"=>57673, 948 | "backend"=>"10.0.199.184", 949 | "backend_port"=>80, 950 | "request_processing_time"=>5.3e-05, 951 | "backend_processing_time"=>0.000913, 952 | "response_processing_time"=>3.6e-05, 953 | "elb_status_code"=>200, 954 | "backend_status_code"=>200, 955 | "received_bytes"=>0, 956 | "sent_bytes"=>3, 957 | "request"=> 958 | "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1", 959 | "user_agent"=>nil, 960 | "ssl_cipher"=>nil, 961 | "ssl_protocol"=>nil, 962 | "request.method"=>"GET", 963 | "request.uri"=> 964 | "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/", 965 | "request.http_version"=>"HTTP/1.1", 966 | "request.uri.scheme"=>"http", 967 | "request.uri.user"=>nil, 968 | "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com", 969 | "request.uri.port"=>80, 970 | "request.uri.path"=>"/", 971 | "request.uri.query"=>nil, 972 | "request.uri.fragment"=>nil}]] 973 | end 974 | 975 | it { is_expected.to match_table expected_emits } 976 | end 977 | 978 | context 'when no user_agent' do 979 | let(:today_access_log) do 980 | <<-EOS 981 | 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol 982 | EOS 983 | end 984 | 985 | before do 986 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 987 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 988 | 989 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do 990 | [double('today_objects', contents: [double('today_object', key: today_object_key)])] 991 | end 992 | 993 | expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do 994 | double('today_s3_object', body: StringIO.new(today_access_log)) 995 | end 996 | 997 | expect(driver.instance).to receive(:save_timestamp).with(today) 998 | expect(driver.instance).to receive(:save_history) 999 | 1000 | expect(CSV).to receive(:parse_line).and_raise('parse error') 1001 | expect(driver.instance).to receive(:unquote).and_raise('unquote error') 1002 | end 1003 | 1004 | specify do 1005 | expect(driver.instance.log).to receive(:warn).with(/parse error: unquote error:/) 1006 | driver_run(driver) 1007 | end 1008 | end 1009 | end 1010 | -------------------------------------------------------------------------------- /spec/in_elb_access_log_client_spec.rb: -------------------------------------------------------------------------------- 1 | describe 'FluentPluginElbAccessLogInput#client' do 2 | let(:account_id) { '123456789012' } 3 | let(:s3_bucket) { 'my-bucket' } 4 | let(:region) { 'us-west-1' } 5 | let(:driver) { create_driver(fluentd_conf) } 6 | let!(:client){ Aws::S3::Client.new(stub_responses: true) } 7 | 8 | let(:fluentd_conf) do 9 | { 10 | account_id: account_id, 11 | s3_bucket: s3_bucket, 12 | region: region, 13 | start_datetime: (today - 1).to_s, 14 | } 15 | end 16 | 17 | let(:today) { Time.parse('2015/05/24 18:30 UTC') } 18 | let(:yesterday) { today - 86400 } 19 | let(:tomorrow) { today + 86400 } 20 | 21 | let(:today_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{today.strftime('%Y/%m/%d')}/" } 22 | let(:yesterday_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{yesterday.strftime('%Y/%m/%d')}/" } 23 | let(:tomorrow_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{tomorrow.strftime('%Y/%m/%d')}/" } 24 | 25 | let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{today.iso8601}_52.68.51.1_8hSqR3o4.log" } 26 | let(:yesterday_object_key) { "#{yesterday_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{yesterday.iso8601}_52.68.51.1_8hSqR3o4.log" } 27 | let(:tomorrow_object_key) { "#{tomorrow_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{tomorrow.iso8601}_52.68.51.1_8hSqR3o4.log" } 28 | 29 | before do 30 | Timecop.freeze(today) 31 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:load_history) { [] } 32 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:parse_tsfile) { nil } 33 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] } 34 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] } 35 | expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] } 36 | allow(FileUtils).to receive(:touch) 37 | expect(driver.instance).to_not receive(:save_timestamp).with(today) 38 | expect(driver.instance).to receive(:save_history) 39 | expect(driver.instance.log).to_not receive(:error) 40 | expect(driver.instance.log).to_not receive(:warn) 41 | end 42 | 43 | after do 44 | Timecop.return 45 | end 46 | 47 | context 'when create client without credentials' do 48 | specify do 49 | expect(Aws::S3::Client).to receive(:new).with( 50 | region: region, 51 | user_agent_suffix: FluentPluginElbAccessLogInput::USER_AGENT_SUFFIX, 52 | ).and_return(client) 53 | 54 | driver_run(driver) 55 | end 56 | end 57 | 58 | context 'when create client with aws_key_id/aws_sec_key' do 59 | let(:aws_key_id) { 'akid' } 60 | let(:aws_sec_key) { 'secret' } 61 | 62 | let(:fluentd_conf) do 63 | { 64 | account_id: account_id, 65 | s3_bucket: s3_bucket, 66 | region: region, 67 | start_datetime: (today - 1).to_s, 68 | aws_key_id: aws_key_id, 69 | aws_sec_key: aws_sec_key, 70 | } 71 | end 72 | 73 | specify do 74 | expect(Aws::S3::Client).to receive(:new).with( 75 | region: region, 76 | user_agent_suffix: FluentPluginElbAccessLogInput::USER_AGENT_SUFFIX, 77 | access_key_id: aws_key_id, 78 | secret_access_key: aws_sec_key, 79 | ).and_return(client) 80 | 81 | driver_run(driver) 82 | end 83 | end 84 | 85 | context 'when create client with profile/credentials_path' do 86 | let(:profile) { 'my-profile' } 87 | let(:credentials_path) { '/foo/bar/zoo' } 88 | 89 | let(:fluentd_conf) do 90 | { 91 | account_id: account_id, 92 | s3_bucket: s3_bucket, 93 | region: region, 94 | start_datetime: (today - 1).to_s, 95 | profile: profile, 96 | credentials_path: credentials_path, 97 | } 98 | end 99 | 100 | specify do 101 | expect(Aws::S3::Client).to receive(:new) do |options| 102 | credentials = options.fetch(:credentials) 103 | expect(credentials.profile_name).to eq profile 104 | expect(credentials.path).to eq credentials_path 105 | client 106 | end 107 | 108 | driver_run(driver) 109 | end 110 | end 111 | 112 | context 'when create client with debug' do 113 | let(:fluentd_conf) do 114 | { 115 | account_id: account_id, 116 | s3_bucket: s3_bucket, 117 | region: region, 118 | start_datetime: (today - 1).to_s, 119 | debug: true, 120 | } 121 | end 122 | 123 | specify do 124 | expect(Aws::S3::Client).to receive(:new) do |options| 125 | expect(options.fetch(:log_level)).to eq :debug 126 | expect(options.fetch(:logger)).to be_a(Logger) 127 | client 128 | end 129 | 130 | driver_run(driver) 131 | end 132 | end 133 | end 134 | -------------------------------------------------------------------------------- /spec/in_elb_access_log_config_spec.rb: -------------------------------------------------------------------------------- 1 | describe 'FluentPluginElbAccessLogInput#configure' do 2 | let(:account_id) { '123456789012' } 3 | let(:s3_bucket) { 'my-bucket' } 4 | let(:region) { 'us-west-1' } 5 | let(:driver) { create_driver(fluentd_conf) } 6 | let(:today) { Time.parse('2015/05/24 18:30 UTC') } 7 | 8 | subject { create_driver(fluentd_conf).instance } 9 | 10 | before do 11 | Timecop.freeze(today) 12 | allow(FileUtils).to receive(:touch) 13 | allow(File).to receive(:read) { nil } 14 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:load_history) { [] } 15 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:parse_tsfile) { nil } 16 | end 17 | 18 | context 'when default' do 19 | let(:fluentd_conf) do 20 | { 21 | account_id: account_id, 22 | s3_bucket: s3_bucket, 23 | region: region, 24 | interval: nil, 25 | } 26 | end 27 | 28 | it do 29 | expect(driver.instance.aws_key_id).to be_nil 30 | expect(driver.instance.aws_sec_key).to be_nil 31 | expect(driver.instance.profile).to be_nil 32 | expect(driver.instance.credentials_path).to be_nil 33 | expect(driver.instance.http_proxy).to be_nil 34 | expect(driver.instance.s3_bucket).to eq s3_bucket 35 | expect(driver.instance.region).to eq region 36 | expect(driver.instance.s3_prefix).to be_nil 37 | expect(driver.instance.tag).to eq 'elb.access_log' 38 | expect(driver.instance.tsfile_path).to eq '/var/tmp/fluent-plugin-elb-access-log.ts' 39 | expect(driver.instance.histfile_path).to eq '/var/tmp/fluent-plugin-elb-access-log.history' 40 | expect(driver.instance.interval).to eq 300 41 | expect(driver.instance.start_datetime).to eq today 42 | expect(driver.instance.buffer_sec).to eq 600 43 | expect(driver.instance.history_length).to eq 100 44 | expect(driver.instance.sampling_interval).to eq 1 45 | expect(driver.instance.debug).to be_falsey 46 | expect(driver.instance.elb_type).to eq 'clb' 47 | expect(driver.instance.filter).to be_nil 48 | expect(driver.instance.filter_operator).to eq 'and' 49 | expect(driver.instance.type_cast).to be_truthy 50 | expect(driver.instance.parse_request).to be_truthy 51 | expect(driver.instance.split_addr_port).to be_truthy 52 | expect(driver.instance.filter).to be_nil 53 | expect(driver.instance.file_filter).to be_nil 54 | expect(driver.instance.request_separator).to eq '.' 55 | end 56 | end 57 | 58 | context 'when pass params' do 59 | let(:aws_key_id) { 'YOUR_ACCESS_KEY_ID' } 60 | let(:aws_sec_key) { 'YOUR_SECRET_ACCESS_KEY' } 61 | let(:profile) { 'PROFILE_NAME' } 62 | let(:credentials_path) { 'path/to/credentials_file' } 63 | let(:http_proxy) { 'http://example.net' } 64 | let(:s3_prefix) { 's3-prefix' } 65 | let(:tag) { 'any.tag' } 66 | let(:tsfile_path) { '/tmp/foo' } 67 | let(:histfile_path) { '/tmp/bar' } 68 | let(:interval) { 500 } 69 | let(:start_datetime) { today - 3600 } 70 | let(:buffer_sec) { 1200 } 71 | let(:history_length) { 200 } 72 | let(:sampling_interval) { 100 } 73 | let(:elb_type) { 'alb' } 74 | let(:filter) { 'elb_status_code:^2' } 75 | let(:filter_operator) { 'or' } 76 | let(:file_filter) { '.*my-elb.*' } 77 | let(:request_separator) { '_' } 78 | 79 | let(:fluentd_conf) do 80 | { 81 | aws_key_id: aws_key_id, 82 | aws_sec_key: aws_sec_key, 83 | profile: profile, 84 | credentials_path: credentials_path, 85 | http_proxy: http_proxy, 86 | account_id: account_id, 87 | s3_bucket: s3_bucket, 88 | region: region, 89 | s3_prefix: s3_prefix, 90 | tag: tag, 91 | tsfile_path: tsfile_path, 92 | histfile_path: histfile_path, 93 | interval: interval, 94 | start_datetime: start_datetime, 95 | buffer_sec: buffer_sec, 96 | history_length: history_length, 97 | sampling_interval: sampling_interval, 98 | debug: true, 99 | elb_type: elb_type, 100 | filter: filter, 101 | filter_operator: filter_operator, 102 | type_cast: 'false', 103 | parse_request: 'false', 104 | split_addr_port: 'false', 105 | file_filter: file_filter, 106 | request_separator: request_separator, 107 | } 108 | end 109 | 110 | it do 111 | expect(driver.instance.aws_key_id).to eq aws_key_id 112 | expect(driver.instance.aws_sec_key).to eq aws_sec_key 113 | expect(driver.instance.profile).to eq profile 114 | expect(driver.instance.credentials_path).to eq credentials_path 115 | expect(driver.instance.http_proxy).to eq http_proxy 116 | expect(driver.instance.s3_bucket).to eq s3_bucket 117 | expect(driver.instance.region).to eq region 118 | expect(driver.instance.s3_prefix).to eq s3_prefix 119 | expect(driver.instance.tag).to eq tag 120 | expect(driver.instance.tsfile_path).to eq tsfile_path 121 | expect(driver.instance.histfile_path).to eq histfile_path 122 | expect(driver.instance.interval).to eq interval 123 | expect(driver.instance.start_datetime).to eq start_datetime 124 | expect(driver.instance.buffer_sec).to eq buffer_sec 125 | expect(driver.instance.history_length).to eq history_length 126 | expect(driver.instance.sampling_interval).to eq sampling_interval 127 | expect(driver.instance.debug).to be_truthy 128 | expect(driver.instance.elb_type).to eq elb_type 129 | expect(driver.instance.filter).to eq('elb_status_code' => /^2/) 130 | expect(driver.instance.filter_operator).to eq filter_operator 131 | expect(driver.instance.type_cast).to be_falsey 132 | expect(driver.instance.parse_request).to be_falsey 133 | expect(driver.instance.split_addr_port).to be_falsey 134 | expect(driver.instance.file_filter).to eq Regexp.new(file_filter) 135 | expect(driver.instance.request_separator).to eq request_separator 136 | end 137 | end 138 | 139 | context 'when start_datetime is set' do 140 | let(:start_datetime) { '2015-01-01 01:02:03 UTC' } 141 | 142 | let(:fluentd_conf) do 143 | { 144 | account_id: account_id, 145 | s3_bucket: s3_bucket, 146 | region: region, 147 | start_datetime: start_datetime 148 | } 149 | end 150 | 151 | it do 152 | expect(driver.instance.start_datetime).to eq Time.parse(start_datetime) 153 | end 154 | end 155 | 156 | context 'when tsfile datetime is set' do 157 | let(:tsfile_datetime) { '2015-02-01 01:02:03 UTC' } 158 | 159 | let(:fluentd_conf) do 160 | { 161 | account_id: account_id, 162 | s3_bucket: s3_bucket, 163 | region: region, 164 | start_datetime: tsfile_datetime 165 | } 166 | end 167 | 168 | before do 169 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:parse_tsfile) { Time.parse(tsfile_datetime) } 170 | end 171 | 172 | it do 173 | expect(driver.instance.start_datetime).to eq Time.parse(tsfile_datetime) 174 | end 175 | end 176 | 177 | context 'when start_datetime and tsfile datetime are set' do 178 | let(:start_datetime) { '2015-01-01 01:02:03 UTC' } 179 | let(:tsfile_datetime) { '2015-02-01 01:02:03 UTC' } 180 | 181 | let(:fluentd_conf) do 182 | { 183 | account_id: account_id, 184 | s3_bucket: s3_bucket, 185 | region: region, 186 | start_datetime: start_datetime 187 | } 188 | end 189 | 190 | before do 191 | allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:parse_tsfile) { Time.parse(tsfile_datetime) } 192 | allow_any_instance_of(Fluent::Test::TestLogger).to receive(:warn).with("start_datetime(#{start_datetime}) is set. but tsfile datetime(#{tsfile_datetime}) is used") 193 | end 194 | 195 | it do 196 | expect(driver.instance.start_datetime).to eq Time.parse(tsfile_datetime) 197 | end 198 | end 199 | 200 | context 'when an invalid ELB type' do 201 | let(:start_datetime) { '2015-01-01 01:02:03 UTC' } 202 | 203 | let(:fluentd_conf) do 204 | { 205 | account_id: account_id, 206 | s3_bucket: s3_bucket, 207 | region: region, 208 | start_datetime: start_datetime, 209 | elb_type: 'invalid', 210 | } 211 | end 212 | 213 | it do 214 | expect { 215 | subject 216 | }.to raise_error 'Invalid ELB type: invalid' 217 | end 218 | end 219 | 220 | context 'when an invalid filter operator' do 221 | let(:start_datetime) { '2015-01-01 01:02:03 UTC' } 222 | 223 | let(:fluentd_conf) do 224 | { 225 | account_id: account_id, 226 | s3_bucket: s3_bucket, 227 | region: region, 228 | start_datetime: start_datetime, 229 | filter_operator: 'invalid', 230 | } 231 | end 232 | 233 | it do 234 | expect { 235 | subject 236 | }.to raise_error 'Invalid filter operator: invalid' 237 | end 238 | end 239 | end 240 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'coveralls' 2 | Coveralls.wear! 3 | 4 | require 'fluent/version' 5 | require 'fluent/test' 6 | 7 | if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('0.14') 8 | require 'fluent/test/driver/input' 9 | end 10 | 11 | require 'fluent/plugin/in_elb_access_log' 12 | 13 | require 'aws-sdk-s3' 14 | require 'time' 15 | require 'timecop' 16 | require 'rspec/match_table' 17 | 18 | # Disable Test::Unit 19 | module Test::Unit::RunCount; def run(*); end; end 20 | 21 | RSpec.configure do |config| 22 | config.before(:all) do 23 | Fluent::Test.setup 24 | end 25 | end 26 | 27 | def create_driver(options = {}) 28 | options = { 29 | interval: 0, 30 | }.merge(options) 31 | 32 | account_id = options.fetch(:account_id) || '123456789012' 33 | s3_bucket = options.fetch(:s3_bucket) || 'my-bucket' 34 | region = options.fetch(:region) || 'us-west-1' 35 | 36 | additional_options = options.select {|k, v| v }.map {|key, value| 37 | "#{key} #{value}" 38 | }.join("\n") 39 | 40 | fluentd_conf = <<-EOS 41 | type elb_access_log 42 | account_id #{account_id} 43 | s3_bucket #{s3_bucket} 44 | region #{region} 45 | #{additional_options} 46 | EOS 47 | 48 | if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('0.14') 49 | Fluent::Test::Driver::Input.new(FluentPluginElbAccessLogInput).configure(fluentd_conf) 50 | else 51 | Fluent::Test::OutputTestDriver.new(FluentPluginElbAccessLogInput).configure(fluentd_conf) 52 | end 53 | end 54 | 55 | def driver_run(driver) 56 | driver.run do 57 | coolio_loop = driver.instance.instance_variable_get(:@loop) 58 | sleep 0.1 until coolio_loop.instance_variable_get(:@running) 59 | sleep 0.1 60 | end 61 | end 62 | 63 | 64 | def driver_events 65 | if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('0.14') 66 | driver.events 67 | else 68 | driver.emits 69 | end 70 | end 71 | 72 | def gzip(str) 73 | io = StringIO.new 74 | 75 | Zlib::GzipWriter.wrap(io) do |gz| 76 | gz << str 77 | end 78 | 79 | io.string 80 | end 81 | 82 | # prevent Test::Unit's AutoRunner from executing during RSpec's rake task 83 | # ref: https://github.com/rspec/rspec-rails/issues/1171 84 | Test::Unit.run = true if defined?(Test::Unit) && Test::Unit.respond_to?(:run=) 85 | --------------------------------------------------------------------------------