├── .gitignore
├── .travis.yml
├── Gemfile
├── LICENSE.txt
├── README.md
├── Rakefile
├── fluent-plugin-elb-log.gemspec
├── fluent.conf.sample
├── lib
    └── fluent
    │   └── plugin
    │       └── in_elb_log.rb
└── test
    ├── helper.rb
    └── plugin
        └── in_elb_log.rb


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.gem
 2 | *.rbc
 3 | .bundle
 4 | .config
 5 | .yardoc
 6 | Gemfile.lock
 7 | InstalledFiles
 8 | _yardoc
 9 | coverage
10 | doc/
11 | lib/bundler/man
12 | pkg
13 | rdoc
14 | spec/reports
15 | test/tmp
16 | test/version_tmp
17 | tmp
18 | .irb_history
19 | .ruby-version
20 | .vscode
21 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | env:
 2 |   global:
 3 |     - CC_TEST_REPORTER_ID=bca5d9771c392e374d213de82dde44181a12b4c8b5d5bd37b249c81ca5bf32f1
 4 |     - GIT_COMMITTED_AT=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then git log -1 --pretty=format:%ct; else git log -1 --skip 1 --pretty=format:%ct; fi)
 5 | language: ruby
 6 | 
 7 | rvm:
 8 |   - 2.1
 9 |   - 2.2
10 |   - 2.3.3
11 |   - 2.4.1
12 | branches:
13 |   only:
14 |     - master
15 | 
16 | before_script:
17 |   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
18 |   - chmod +x ./cc-test-reporter
19 | after_script:
20 |   - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
21 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | 
3 | # Specify your gem's dependencies in fluent-plugin-elb-log.gemspec
4 | gemspec
5 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2014 shinsaka
 2 | 
 3 | MIT License
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining
 6 | a copy of this software and associated documentation files (the
 7 | "Software"), to deal in the Software without restriction, including
 8 | without limitation the rights to use, copy, modify, merge, publish,
 9 | distribute, sublicense, and/or sell copies of the Software, and to
10 | permit persons to whom the Software is furnished to do so, subject to
11 | the following conditions:
12 | 
13 | The above copyright notice and this permission notice shall be
14 | included in all copies or substantial portions of the Software.
15 | 
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Amazon ELB log input plugin for fluentd
  2 | 
  3 | [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elb-log.svg)](https://badge.fury.io/rb/fluent-plugin-elb-log)
  4 | [![Build Status](https://travis-ci.org/shinsaka/fluent-plugin-elb-log.svg?branch=master)](https://travis-ci.org/shinsaka/fluent-plugin-elb-log)
  5 | [![Code Climate](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log/badges/gpa.svg)](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log)
  6 | [![Test Coverage](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log/badges/coverage.svg)](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log/coverage)
  7 | 
  8 | ## Overview
  9 | - Amazon Web Services ELB log input plubin for fluentd
 10 | 
 11 | ## Requirements
 12 | 
 13 | | fluent-plugin-elb-log | fluentd    | ruby   |
 14 | |-----------------------|------------|--------|
 15 | | >= 0.3.0              | >= v0.14.0 | >= 2.1 |
 16 | | < 0.3.0               | >= v0.12.0 | >= 1.9 |
 17 | 
 18 | ## Installation
 19 | 
 20 |     $ fluentd-gem fluent-plugin-elb-log
 21 | 
 22 | ## AWS ELB Settings
 23 | - settings see: [Elastic Load Balancing](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html)
 24 | - developer guide: [](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html)
 25 | 
 26 | ## Different from version 0.4.x
 27 | - Using version 3 of the AWS SDK for Ruby.
 28 | 
 29 | ## Support Application Load Balancer (ver 0.4.0 or later)
 30 | - Support Access Logs for Application Load Balancer
 31 |  - https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html
 32 | - Existing ELB is called Classic Load Balancer
 33 |  - http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html
 34 | 
 35 | ## When SSL certification error
 36 | log:
 37 | ```
 38 | SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed
 39 | ```
 40 | Do env setting follows:
 41 | ```
 42 | SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
 43 | ```
 44 | 
 45 | ## Configuration
 46 | 
 47 | ```config
 48 | <source>
 49 |   @type elb_log
 50 | 
 51 |   # following attibutes are required
 52 |   region            <region name>
 53 |   s3_bucketname     <bucketname>
 54 |   s3_prefix         <elb log's prefix>
 55 |   timestamp_file    <proc last file timestamp record filename>
 56 |   buf_file          <buffer file path>
 57 |   refresh_interval  <interval number by second>
 58 |   tag               <tag name(default: elb.access)>
 59 |   delete            <boolean delete processed log files from S3(default: false)>
 60 |   include_all_message <boolean (default:false)>
 61 |   start_time        <last timestamp to start from>
 62 |   exclude_pattern_logfile_elb_name <exclude pattern for logfile_elb_name key>
 63 |   use_sqs           <boolean use SQS polling instead of iteratively processing the whole bucket>
 64 | 
 65 |   # following attibutes are required if you don't use IAM Role
 66 |   access_key_id     <access_key>
 67 |   secret_access_key <secret_access_key>
 68 | </source>
 69 | ```
 70 | 
 71 | `use_sqs` automatically creates an SQS queue named `fluent-plugin-elb-log-<current_timestamp>` 
 72 | and sets up the `all object create event` S3 event notification for the chosen S3 bucket. Stopping fluentd deletes both autmatically.
 73 | To make it work, the following IAM policy should be attached to the instance IAM role or the user:
 74 | ```
 75 | {
 76 |     "Version": "2012-10-17",
 77 |     "Statement": [
 78 |         {
 79 |             "Sid": "FluentdPermissions",
 80 |             "Effect": "Allow",
 81 |             "Action": [
 82 |                 "sqs:DeleteMessage",
 83 |                 "s3:GetObject",
 84 |                 "sqs:ReceiveMessage",
 85 |                 "sqs:DeleteQueue",
 86 |                 "sqs:GetQueueAttributes",
 87 |                 "s3:ListBucket",
 88 |                 "s3:PutBucketNotification",
 89 |                 "sqs:CreateQueue"
 90 |             ],
 91 |             "Resource": [
 92 |                 "arn:aws:sqs:*:123456789012:fluent-plugin-elb-log-*",
 93 |                 "arn:aws:s3:::alb-logs-bucket/*",
 94 |                 "arn:aws:s3:::alb-logs-bucket"
 95 |             ]
 96 |         }
 97 |     ]
 98 | }
 99 | ```
100 | When `use_sqs` is false:
101 |   - 300 seconds is a good value for the `refresh_interval`
102 |   - the plugin executes processing the whole S3 bucket (with the respect of `s3_prefix` and `start_time`/`timestamp_file`) every `refresh_interval`
103 | 
104 | When `use_sqs` is true:
105 |   - `refresh_interval` of 30-60 seconds should be fine.
106 |   - the plugin executes processing the whole S3 bucket (with the respect of `s3_prefix` and `start_time`/`timestamp_file`) only once (at start) and then 
107 | polls the SQS queue every `refresh_interval`.
108 | 
109 | ### Example setting
110 | ```config
111 | <source>
112 |   @type elb_log
113 |   region            us-east-1
114 |   s3_bucketname     my-elblog-bucket
115 |   s3_prefix         prodcution/web
116 |   timestamp_file    /tmp/elb_last_at.dat
117 |   buf_file          /tmp/fluentd-elblog.tmpfile
118 |   refresh_interval  30
119 |   tag               elb.access
120 |   delete            false
121 |   include_all_message false
122 |   exclude_pattern_logfile_elb_name "^app\.(uat|qa)\."
123 |   start_time        2025-02-27T10:45:00
124 |   use_sqs           true
125 |   access_key_id     XXXXXXXXXXXXXXXXXXXX
126 |   secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
127 | </source>
128 | 
129 | <filter elb.access>
130 |   @type record_transformer
131 |   <record>
132 |     timestamp ${record["request_creation_time"]}
133 |     logfile_name ${record["key"]}
134 |   </record>
135 |   remove_keys prefix,logfile_date,logfile_elb_name,logfile_hash,logfile_timestamp,logfile_timestamp_unixtime,key,time,s3_last_modified_unixtime
136 | </filter>
137 | 
138 | <match elb.access>
139 | #  @type stdout
140 |   @type opensearch
141 |   hosts node-1,node-2,node-3
142 |   port 9200
143 |   scheme https
144 |   logstash_format true
145 |   logstash_prefix alb-logs
146 |   user fluentd
147 |   password secret
148 |   ssl_verify true
149 |   ca_file /etc/fluentd/root-ca.pem
150 |   flush_interval 300s
151 | </match>
152 | ```
153 | 
154 | ### json output example
155 | ```json
156 | {
157 |     "account_id":"123456789012",
158 |     "region":"ap-northeast-1",
159 |     "logfile_date":"2015/06/15",
160 |     "logfile_elb_name":"my-elb-name",
161 |     "elb_ip_address":"52.0.0.0",
162 |     "logfile_hash":"12squv5w",
163 |     "logfile_timestamp":"20150615T0400Z",
164 |     "key":"TEST/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2015/06/15/123456789012_elasticloadbalancing_ap-northeast-1_my-elb-name_20150615T0400Z_52.68.215.138_69squv5w.log",
165 |     "prefix":"TEST",
166 |     "elb_timestamp_unixtime":1434340800,
167 |     "time":"2015-06-15T03:47:12.728427+0000",
168 |     "elb":"my-elb-name",
169 |     "client":"54.1.1.1",
170 |     "client_port":"43759",
171 |     "target":"10.0.0.1",
172 |     "target_port":"80",
173 |     "request_processing_time":4.0e-05,
174 |     "target_processing_time":0.105048,
175 |     "response_processing_time":2.4e-05,
176 |     "elb_status_code":"200",
177 |     "target_status_code":"200",
178 |     "received_bytes":0,
179 |     "sent_bytes":4622,
180 |     "request_method":"GET",
181 |     "request_uri":"https://my-elb-test.example.com/",
182 |     "request_protocol":"HTTP/1.1",
183 |     "user_agent":"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
184 |     "ssl_cipher":"DHE-RSA-AES128-SHA",
185 |     "ssl_protocol":"TLSv1.2",
186 |     "type":"http",
187 |     "target_group_arn": "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d",
188 |     "trace_id": "\"Root=1-xxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx\"",
189 |     "domain_name": "-",
190 |     "chosen_cert_arn": "-",
191 |     "matched_rule_priority": "0",
192 |     "request_creation_time": "2099-10-26T06:10:03.050000Z",
193 |     "actions_executed": "forward",
194 |     "redirect_url": "-",
195 |     "error_reason": "-",
196 |     "target_port_list": "\"192.168.0.1:443\"",
197 |     "target_status_code_list": "\"301\"",
198 |     "classification": "-",
199 |     "classification_reason": "-",
200 |     "conn_trace_id": "TID_xxxxxxxxxxxxxxxxxxxxxxx"
201 | }
202 | ```
203 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | require "bundler/gem_tasks"
 2 | 
 3 | require 'rake/testtask'
 4 | Rake::TestTask.new(:test) do |test|
 5 |   test.libs << 'lib' << 'test'
 6 |   test.pattern = 'test/**/*.rb'
 7 |   test.verbose = true
 8 |   test.warning = false
 9 | end
10 | 
11 | desc 'Run tests for all'
12 | task :default => :test
13 | 


--------------------------------------------------------------------------------
/fluent-plugin-elb-log.gemspec:
--------------------------------------------------------------------------------
 1 | # coding: utf-8
 2 | lib = File.expand_path('../lib', __FILE__)
 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 4 | 
 5 | Gem::Specification.new do |spec|
 6 |   spec.name          = "fluent-plugin-elb-log"
 7 |   spec.version       = "1.4.0"
 8 |   spec.authors       = ["shinsaka","jazzl0ver"]
 9 |   spec.email         = ["shinx1265@gmail.com"]
10 |   spec.summary       = "Amazon ELB log input plugin"
11 |   spec.description   = "Amazon ELB log input plugin for fluentd"
12 |   spec.homepage      = "https://github.com/jazzl0ver/fluent-plugin-elb-log"
13 |   spec.license       = "MIT"
14 | 
15 |   spec.files         = `git ls-files -z`.split("\x0")
16 |   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17 |   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
18 |   spec.require_paths = ["lib"]
19 | 
20 |   spec.add_dependency "fluentd", ">= 0.14.0", "< 2"
21 |   spec.add_dependency "aws-sdk-s3", "~> 1"
22 |   spec.add_dependency "aws-sdk-ec2", "~> 1"
23 | 
24 |   spec.add_development_dependency "bundler", ">=1.17"
25 |   spec.add_development_dependency "rake", "~> 12"
26 |   spec.add_development_dependency "test-unit", "~> 3.2"
27 |   spec.add_development_dependency "webmock", "~>3"
28 |   spec.add_development_dependency "simplecov", "~>0"
29 | end
30 | 


--------------------------------------------------------------------------------
/fluent.conf.sample:
--------------------------------------------------------------------------------
 1 | <source>
 2 |   @type elb_log
 3 |   region            us-east-1
 4 |   s3_bucketname     my-elblog-bucket
 5 |   s3_prefix         prodcution/web
 6 |   timestamp_file    /tmp/elb_last_at.dat
 7 |   buf_file          /tmp/fluentd-elblog.tmpfile
 8 |   refresh_interval  30
 9 |   tag               elb.access
10 |   use_sqs           true
11 |   access_key_id     XXXXXXXXXXXXXXXXXXXX
12 |   secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
13 | </source>
14 | 
15 | <filter elb.access>
16 |   @type record_transformer
17 |   <record>
18 |     timestamp ${record["request_creation_time"]}
19 |     logfile_name ${record["key"]}
20 |   </record>
21 |   remove_keys prefix,logfile_date,logfile_elb_name,logfile_hash,logfile_timestamp,logfile_timestamp_unixtime,key,time,s3_last_modified_unixtime
22 | </filter>
23 | 
24 | <match elb.access>
25 |     @type stdout
26 | </match>
27 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/in_elb_log.rb:
--------------------------------------------------------------------------------
  1 | require 'time'
  2 | require 'zlib'
  3 | require 'fileutils'
  4 | require 'aws-sdk-s3'
  5 | require 'aws-sdk-ec2'
  6 | require 'aws-sdk-sqs'
  7 | require 'fluent/input'
  8 | require 'digest/sha1'
  9 | 
 10 | class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
 11 |   Fluent::Plugin.register_input('elb_log', self)
 12 | 
 13 |   helpers :timer
 14 | 
 15 |   LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<logfile_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log(.gz)?$/
 16 |   ACCESSLOG_REGEXP = /^((?<type>[a-z0-9]+) )?(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>\S+) (?<client>\S+)\:(?<client_port>\S+) (?<target>[^:\s]+)(?::(?<target_port>\S+))? (?<request_processing_time>\S+) (?<target_processing_time>\S+) (?<response_processing_time>\S+) (?<elb_status_code>\S+) (?<target_status_code>\S+) (?<received_bytes>\S+) (?<sent_bytes>\S+) \"(?<request_method>\S+) (?<request_uri>\S+) (?<request_protocol>\S+)\" \"(?<user_agent>.+?)\" (?<ssl_cipher>\S+) (?<ssl_protocol>\S+) (?<target_group_arn>\S+) \"(?<trace_id>\S+)\" \"(?<domain_name>\S+)\" \"(?<chosen_cert_arn>\S+)\" (?<matched_rule_priority>\S+) (?<request_creation_time>\S+) \"(?<actions_executed>\S+)\" \"(?<redirect_url>\S+)\" \"(?<error_reason>\S+)\" \"(?<target_port_list>\S+)\" \"(?<target_status_code_list>\S+)\" \"(?<classification>\S+)\" \"(?<classification_reason>\S+)\" (?<conn_trace_id>\S+)/
 17 |   config_param :access_key_id, :string, default: nil, secret: true
 18 |   config_param :secret_access_key, :string, default: nil, secret: true
 19 |   config_param :region, :string
 20 |   config_param :s3_bucketname, :string, default: nil
 21 |   config_param :s3_prefix, :string, default: nil
 22 |   config_param :tag, :string, default: 'elb.access'
 23 |   config_param :timestamp_file, :string, default: nil
 24 |   config_param :refresh_interval, :integer, default: 300
 25 |   config_param :buf_file, :string, default: './fluentd_elb_log_buf_file'
 26 |   config_param :http_proxy, :string, default: nil
 27 |   config_param :start_time, :string, default: nil
 28 |   config_param :delete, :bool, default: false
 29 |   config_param :num_nodes, :integer, default: 1
 30 |   config_param :node_no, :integer, default: 0
 31 |   config_param :include_all_message, :bool, default: false
 32 |   config_param :exclude_pattern_logfile_elb_name, :string, default: nil
 33 |   config_param :use_sqs, :bool, default: true
 34 | 
 35 |   def configure(conf)
 36 |     super
 37 | 
 38 |     if !has_iam_role?
 39 |       raise Fluent::ConfigError.new("access_key_id is required") if @access_key_id.nil?
 40 |       raise Fluent::ConfigError.new("secret_access_key is required") if @secret_access_key.nil?
 41 |     end
 42 |     raise Fluent::ConfigError.new("s3_bucketname is required") unless @s3_bucketname
 43 |     raise Fluent::ConfigError.new("timestamp_file is required") unless @timestamp_file
 44 | 
 45 |     @s3_client = s3_client
 46 |     raise Fluent::ConfigError.new("s3 bucket not found #{@s3_bucketname}") unless s3bucket_is_ok?
 47 |   end
 48 | 
 49 |   def initialize
 50 |     super
 51 |     @running = true
 52 |   end
 53 | 
 54 |   def start
 55 |     super
 56 | 
 57 |     # files touch
 58 |     File.open(@timestamp_file, File::RDWR|File::CREAT).close
 59 |     File.open(@buf_file, File::RDWR|File::CREAT).close
 60 | 
 61 |     @exclude_pattern_logfile_elb_name_re = Regexp.new(@exclude_pattern_logfile_elb_name)
 62 | 
 63 |     Signal.trap('INT') { shutdown }
 64 | 
 65 |     if @use_sqs
 66 |       input
 67 |       @sqs_client = sqs_client
 68 |       setup_sqs_timer
 69 |     else
 70 |       setup_input_timer
 71 |     end
 72 |   end
 73 | 
 74 |   private
 75 | 
 76 |   def shutdown
 77 |     log.debug "shutdown"
 78 |     if @running
 79 | 	@running = false
 80 |         if @use_sqs
 81 |           log.debug "pausing shutdown for 2x#{@refresh_interval}"
 82 |           sleep 2*@refresh_interval
 83 |           unset_sqs if @queue_url && @queue_url != ""
 84 |         else
 85 |           log.debug "pausing shutdown for 30 seconds"
 86 |           sleep 30
 87 |         end
 88 |     end
 89 |   end
 90 | 
 91 |   def setup_sqs_timer
 92 |     if @running
 93 |       setup_sqs
 94 |       timer_execute(:in_elb_log, @refresh_interval) do
 95 | 	if @running
 96 |     	    process_sqs
 97 |     	    log.debug "sleeping for #{@refresh_interval}"
 98 |     	else
 99 |     	    log.debug "Unsetting SQS"
100 | 	    unset_sqs if @queue_url && @queue_url != ""
101 | 	end
102 |       end
103 |     end
104 |   end
105 | 
106 |   def setup_input_timer
107 |     if @running
108 |       timer_execute(:in_elb_log, @refresh_interval) do
109 | 	if @running
110 |     	    input
111 | 	    log.debug "sleeping input for #{@refresh_interval}"
112 | 	end
113 |       end
114 |     end
115 |   end
116 | 
117 |   def has_iam_role?
118 |     begin
119 |       ec2 = Aws::EC2::Client.new(region: @region)
120 |       !ec2.config.credentials.nil?
121 |     rescue => e
122 |       log.warn "EC2 Client error occurred: #{e.message}"
123 |     end
124 |   end
125 | 
126 |   def setup_sqs
127 |     begin
128 |     timestamp = Time.now.to_i
129 |     queue_name = "fluent-plugin-elb-log-#{timestamp}"
130 | 
131 |     sts_client = Aws::STS::Client.new
132 |     account_id = sts_client.get_caller_identity.account
133 | 
134 |     queue_policy = {
135 |       "Version": "2012-10-17",
136 |       "Id": "__default_policy_ID",
137 |       "Statement": [
138 |         {
139 |           "Sid": "S3_service_publish",
140 |           "Effect": "Allow",
141 |           "Principal": {
142 |             "Service": "s3.amazonaws.com"
143 |           },
144 |           "Action": "SQS:SendMessage",
145 |           "Resource": "arn:aws:sqs:us-east-1:#{account_id}:#{queue_name}",
146 |           "Condition": {
147 |             "StringEquals": {
148 |               "aws:SourceAccount": "#{account_id}"
149 | 	    },
150 |             "ArnLike": {
151 |     	      "aws:SourceArn": "arn:aws:s3:*:*:#{@s3_bucket}"
152 | 	    }
153 |           }
154 |         }
155 |       ]
156 |     }.to_json
157 |     create_queue_response = @sqs_client.create_queue(queue_name: queue_name,
158 |       attributes: {
159 |         'Policy' => queue_policy
160 |       }
161 |     )
162 |     @queue_url = create_queue_response.queue_url
163 |     queue_attributes = @sqs_client.get_queue_attributes(
164 |       queue_url: @queue_url,
165 |       attribute_names: ['QueueArn']
166 |     )
167 |     queue_arn = queue_attributes.attributes['QueueArn']
168 |     log.debug "New SQS queue created: #{queue_arn}"
169 | 
170 |     notification_configuration = {
171 |       queue_configurations: [
172 | 	{
173 | 	  id: "fluent-plugin-elb-log",
174 |           events: ['s3:ObjectCreated:*'],
175 |           queue_arn: "#{queue_arn}",
176 |           filter: {
177 |             key: {
178 |               filter_rules: [
179 |                 {
180 |                   name: 'Prefix',
181 |                   value: "#{@s3_prefix}"
182 |                 }
183 |               ]
184 |             }
185 |           }
186 |         }
187 |       ]
188 |     }
189 | 
190 |     @s3_client.put_bucket_notification_configuration(
191 |       bucket: @s3_bucketname,
192 |       notification_configuration: notification_configuration
193 |     )
194 |     log.debug "S3 notification events has been set for #{@s3_bucketname}/#{@s3_prefix}"
195 | 
196 |     rescue Aws::SQS::Errors::InvalidAttributeValue => e
197 |       log.debug "SQS error: #{e.message}"
198 | 
199 |     rescue Aws::S3::Errors::InvalidArgument => e
200 |       log.debug "S3 Event error: #{e.message}"
201 |       sqs_client.delete_queue(queue_url: @queue_url)
202 |       log.debug "#{queue_arn} deleted"
203 |     end
204 |   end
205 | 
206 |   def unset_sqs
207 |     return if @queue_url.nil? || @queue_url == ""
208 |     @s3_client.put_bucket_notification_configuration(
209 |       bucket: @s3_bucketname,
210 |       notification_configuration: {}
211 |     )
212 |     log.debug "S3 notification events has been removed"
213 | 
214 |     @sqs_client.delete_queue(queue_url: @queue_url)
215 |     log.debug "SQS queue #{@queue_url} has been removed"
216 |     @queue_url = ""
217 |   end
218 | 
219 |   def get_timestamp_file
220 |     begin
221 |       # get timestamp last proc
222 |       start_time = @start_time ? Time.parse(@start_time).utc : Time.at(0)
223 |       timestamp = start_time.to_i
224 |       log.debug "timestamp file #{@timestamp_file} read"
225 |       File.open(@timestamp_file, File::RDONLY) do |file|
226 |         if file.size > 0
227 |           timestamp_from_file = file.read.to_i
228 |           if timestamp_from_file > timestamp
229 |             timestamp = timestamp_from_file
230 |           end
231 |         end
232 |       end
233 |       log.debug "timestamp start at:" + Time.at(timestamp).to_s
234 |       return timestamp
235 |     rescue => e
236 |       log.warn "timestamp file get and parse error occurred: #{e.message}"
237 |     end
238 |   end
239 | 
240 |   def put_timestamp_file(timestamp)
241 |     begin
242 |       log.debug "timestamp file #{@timestamp_file} write"
243 |       File.open(@timestamp_file, File::WRONLY|File::CREAT|File::TRUNC) do |file|
244 |         file.puts timestamp.to_s
245 |       end
246 |     rescue => e
247 |       log.warn "timestamp file get and parse error occurred: #{e.message}"
248 |     end
249 |   end
250 | 
251 |   def s3_client
252 |     begin
253 |       options = {
254 |         region: @region,
255 |       }
256 |       if @access_key_id && @secret_access_key
257 |         options[:access_key_id] = @access_key_id
258 |         options[:secret_access_key] = @secret_access_key
259 |       end
260 |       if @http_proxy
261 |         options[:http_proxy] = @http_proxy
262 |       end
263 |       log.debug "S3 client connect"
264 |       Aws::S3::Client.new(options)
265 |     rescue => e
266 |       log.warn "S3 Client error occurred: #{e.message}"
267 |     end
268 |   end
269 | 
270 |   def s3bucket_is_ok?
271 |     log.debug "searching for bucket #{@s3_bucketname}"
272 | 
273 |     begin
274 |       # try get one
275 |       !(get_object_list(1).nil?)
276 |     rescue => e
277 |       log.warn "error occurred: #{e.message}"
278 |       false
279 |     end
280 |   end
281 | 
282 |   def sqs_client
283 |     begin
284 |       options = {
285 |         region: @region,
286 |       }
287 |       if @access_key_id && @secret_access_key
288 |         options[:access_key_id] = @access_key_id
289 |         options[:secret_access_key] = @secret_access_key
290 |       end
291 |       if @http_proxy
292 |         options[:http_proxy] = @http_proxy
293 |       end
294 |       log.debug "SQS client connect"
295 |       Aws::SQS::Client.new(options)
296 |     rescue => e
297 |       log.warn "SQS Client error occurred: #{e.message}"
298 |     end
299 |   end
300 | 
301 |   def input
302 |     begin
303 |       log.debug "input start"
304 |       timestamp = get_timestamp_file()
305 | 
306 |       object_keys = get_object_keys(timestamp)
307 |       object_keys = sort_object_key(object_keys)
308 | 
309 |       log.info "found #{object_keys.count} new object(s)."
310 | 
311 |       object_keys.each do |object_key|
312 |         record_common = {
313 |           "account_id" => object_key[:account_id],
314 |           "region" => object_key[:region],
315 |           "logfile_date" => object_key[:logfile_date],
316 |           "logfile_elb_name" => object_key[:logfile_elb_name],
317 |           "elb_ip_address" => object_key[:elb_ip_address],
318 |           "logfile_hash" => object_key[:logfile_hash],
319 |           "logfile_timestamp" => object_key[:logfile_timestamp],
320 |           "key" => object_key[:key],
321 |           "prefix" => object_key[:prefix],
322 |           "logfile_timestamp_unixtime" => object_key[:logfile_timestamp_unixtime],
323 |           "s3_last_modified_unixtime" => object_key[:s3_last_modified_unixtime],
324 |         }
325 | 
326 |         get_file_from_s3(object_key[:key])
327 |         emit_lines_from_buffer_file(record_common)
328 | 
329 |         put_timestamp_file(object_key[:s3_last_modified_unixtime])
330 | 
331 |         if @delete
332 |           delete_file_from_s3(object_key[:key])
333 |         end
334 |       end
335 |     rescue => e
336 |       log.warn "error occurred: #{e.message}"
337 |     end
338 |   end
339 | 
340 |   def process_sqs
341 |     begin
342 |       if @running
343 |         number_of_messages = @sqs_client.get_queue_attributes(
344 |           queue_url: @queue_url,
345 |           attribute_names: ["ApproximateNumberOfMessages"]
346 |         ).attributes["ApproximateNumberOfMessages"].to_i
347 | 
348 | 	count = 0
349 | 	while count < number_of_messages
350 | 	    messages = @sqs_client.receive_message(
351 |     	      queue_url: @queue_url,
352 |               max_number_of_messages: 10,
353 |               wait_time_seconds: 10
354 |     	    ).messages
355 |     	    log.debug "SQS queue has ApproximateNumberOfMessages=#{number_of_messages}" if number_of_messages > 0
356 | 
357 | 	    break if messages.empty?
358 | 
359 |     	    messages.each do |message|
360 |     	      count += 1
361 |     	      s3_event = JSON.parse(message.body)
362 |               unless s3_event.key?('Records')
363 |         	@sqs_client.delete_message(
364 |                  queue_url: @queue_url,
365 |                  receipt_handle: message.receipt_handle
366 | 	        )
367 | 	        next
368 | 	      end
369 |               #log.debug "S3 event: #{s3_event}"
370 |               bucket = s3_event['Records'][0]['s3']['bucket']['name']
371 |               key = s3_event['Records'][0]['s3']['object']['key']
372 |               event_time = Time.parse(s3_event['Records'][0]['eventTime']).to_i
373 |               log.debug "S3 event received for #{key} at #{s3_event['Records'][0]['eventTime']}"
374 | 
375 |               object_key = get_object_key(key, event_time, 0)
376 |               if object_key.nil?
377 |         	@sqs_client.delete_message(
378 |                  queue_url: @queue_url,
379 |                  receipt_handle: message.receipt_handle
380 | 	        )
381 | 	        next
382 | 	      end
383 | 
384 |               record_common = {
385 |                 "account_id" => object_key[:account_id],
386 |                 "region" => object_key[:region],
387 |                 "logfile_date" => object_key[:logfile_date],
388 |                 "logfile_elb_name" => object_key[:logfile_elb_name],
389 |                 "elb_ip_address" => object_key[:elb_ip_address],
390 |                 "logfile_hash" => object_key[:logfile_hash],
391 |                 "logfile_timestamp" => object_key[:logfile_timestamp],
392 |                 "key" => object_key[:key],
393 |                 "prefix" => object_key[:prefix],
394 |                 "logfile_timestamp_unixtime" => object_key[:logfile_timestamp_unixtime],
395 |                 "s3_last_modified_unixtime" => object_key[:s3_last_modified_unixtime],
396 |               }
397 | 
398 |               get_file_from_s3(object_key[:key])
399 |               emit_lines_from_buffer_file(record_common)
400 | 
401 |               put_timestamp_file(object_key[:s3_last_modified_unixtime])
402 | 
403 |               if @delete
404 |                 delete_file_from_s3(object_key[:key])
405 |               end
406 | 
407 |               @sqs_client.delete_message(
408 |                queue_url: @queue_url,
409 |                receipt_handle: message.receipt_handle
410 |               )
411 |     	    end
412 |     	    if count == 0
413 |     		log.debug "No messages out of #{number_of_messages} were processed - should never happen"
414 |     		break
415 |     	    end
416 |         end
417 |       end
418 |       rescue => e
419 |         log.warn "error occurred: #{e.message}"
420 |     end
421 |   end
422 | 
423 |   def sort_object_key(src_object_keys)
424 |     begin
425 |       src_object_keys.sort do |a, b|
426 |         a[:s3_last_modified_unixtime] <=> b[:s3_last_modified_unixtime]
427 |       end
428 |     rescue => e
429 |       log.warn "error occurred: #{e.message}"
430 |     end
431 |   end
432 | 
433 |   def get_object_list(max_num)
434 |     @s3_client.list_objects(
435 |       bucket: @s3_bucketname,
436 |       max_keys: max_num,
437 |       prefix: @s3_prefix
438 |     )
439 |   end
440 | 
441 |   def get_object_key(key, last_modified, timestamp)
442 |     node_no = Digest::SHA1.hexdigest(key).to_i(16) % @num_nodes
443 |     return nil unless node_no == @node_no
444 | 
445 |     matches = LOGFILE_REGEXP.match(key)
446 | 
447 |     s3_last_modified_unixtime = last_modified
448 |     if s3_last_modified_unixtime > timestamp and matches
449 |       exclude_pattern_matches = @exclude_pattern_logfile_elb_name_re.match(matches[:logfile_elb_name])
450 |       if exclude_pattern_matches
451 |         log.debug "Skipping object #{key} b/c it matches exclude_pattern_logfile_elb_name"
452 |         return nil
453 |       end
454 | 
455 |       object_key = {
456 |         key: key,
457 |         prefix: matches[:prefix],
458 |         account_id: matches[:account_id],
459 |         region: matches[:region],
460 |         logfile_date: matches[:logfile_date],
461 |         logfile_elb_name: matches[:logfile_elb_name],
462 |         logfile_timestamp: matches[:logfile_timestamp],
463 |         elb_ip_address: matches[:elb_ip_address],
464 |         logfile_hash: matches[:logfile_hash],
465 |         logfile_timestamp_unixtime: Time.parse(matches[:logfile_timestamp]).to_i,
466 |         s3_last_modified_unixtime: s3_last_modified_unixtime,
467 |       }
468 |     end
469 | 
470 |     return object_key
471 |   end
472 | 
473 |   def get_object_keys(timestamp)
474 |     object_keys = []
475 | 
476 |     resp = @s3_client.list_objects_v2(
477 |       bucket: @s3_bucketname,
478 |       prefix: @s3_prefix
479 |     )
480 | 
481 |     while @running do
482 |       resp.contents.each do |content|
483 |         log.debug "Getting #{content.key}"
484 |         object_key = get_object_key(content.key, content.last_modified.to_i, timestamp)
485 |         next if object_key.nil?
486 |         object_keys << object_key
487 |       end
488 | 
489 |       if !resp.is_truncated
490 |         return object_keys
491 |       end
492 | 
493 |       resp = @s3_client.list_objects_v2(
494 |         bucket: @s3_bucketname,
495 |         prefix: @s3_prefix,
496 |         continuation_token: resp.next_continuation_token
497 |       )
498 |     end
499 | 
500 |     return object_keys
501 |   end
502 | 
503 |   def inflate(srcfile, dstfile)
504 |     File.open(dstfile, File::WRONLY|File::CREAT|File::TRUNC) do |bfile|
505 |       File.open(srcfile) do |file|
506 |         zio = file
507 |         loop do
508 |           io = Zlib::GzipReader.new zio
509 |           bfile.write io.read
510 |           unused = io.unused
511 |           io.finish
512 |           break if unused.nil?
513 |           zio.pos -= unused.length
514 |         end
515 |       end
516 |     end
517 |   end
518 | 
519 |   def get_file_from_s3(object_name)
520 |     begin
521 |       log.debug "retrieving #{object_name}"
522 | 
523 |       Tempfile.create('fluent-elblog') do |tfile|
524 |         @s3_client.get_object(bucket: @s3_bucketname, key: object_name, response_target: tfile.path)
525 | 
526 |         if File.extname(object_name) != '.gz'
527 |           FileUtils.cp(tfile.path, @buf_file)
528 |         else
529 |           inflate(tfile.path, @buf_file)
530 |         end
531 |       end
532 |     rescue => e
533 |       log.warn "error occurred: #{e.message}, #{e.backtrace}"
534 |     end
535 |   end
536 | 
537 |   def delete_file_from_s3(object_name)
538 |     begin
539 |       log.debug "deleting object from s3 name is #{object_name}"
540 | 
541 |       @s3_client.delete_object(bucket: @s3_bucketname, key: object_name)
542 |     rescue => e
543 |       log.warn "error occurred: #{e.message}, #{e.backtrace}"
544 |     end
545 |   end
546 | 
547 |   def emit_lines_from_buffer_file(record_common)
548 |     begin
549 |       # emit per line
550 |       File.open(@buf_file, File::RDONLY) do |file|
551 |         file.each_line do |line|
552 |           line_match = ACCESSLOG_REGEXP.match(line)
553 |           unless line_match
554 |             log.info "nomatch log found: #{line} in #{record_common['key']}"
555 |             next
556 |           end
557 | 
558 |           now = Fluent::Engine.now
559 |           time = Time.parse(line_match[:time]).to_i rescue now
560 | 
561 |           router.emit(
562 |             @tag,
563 |             time,
564 |             record_common
565 |               .merge(format_record(line_match)
566 |               .merge(@include_all_message ? {"all_message" => line} : {})
567 |             )
568 |           )
569 |         end
570 |       end
571 |     rescue => e
572 |       log.warn "error occurred: #{e.message}"
573 |     end
574 |   end
575 | 
576 |   def format_record(item)
577 |     { "time" => item[:time].gsub(/Z/, '+0000'),
578 |       "elb" => item[:elb],
579 |       "client" => item[:client],
580 |       "client_port" => item[:client_port],
581 |       "target" => item[:target],
582 |       "target_port" => item[:target_port],
583 |       "request_processing_time" => item[:request_processing_time].to_f,
584 |       "target_processing_time" => item[:target_processing_time].to_f,
585 |       "response_processing_time" => item[:response_processing_time].to_f,
586 |       "elb_status_code" => item[:elb_status_code],
587 |       "target_status_code" => item[:target_status_code],
588 |       "received_bytes" => item[:received_bytes].to_i,
589 |       "sent_bytes" => item[:sent_bytes].to_i,
590 |       "request_method" => item[:request_method],
591 |       "request_uri" => item[:request_uri],
592 |       "request_protocol" => item[:request_protocol],
593 |       "user_agent" => item[:user_agent],
594 |       "ssl_cipher" => item[:ssl_cipher],
595 |       "ssl_protocol" => item[:ssl_protocol],
596 |       "type" => item[:type],
597 |       "target_group_arn" => item[:target_group_arn],
598 |       "trace_id" => item[:trace_id],
599 |       "domain_name" => item[:domain_name],
600 |       "chosen_cert_arn" => item[:chosen_cert_arn],
601 |       "matched_rule_priority" => item[:matched_rule_priority],
602 |       "request_creation_time" => item[:request_creation_time],
603 |       "actions_executed" => item[:actions_executed],
604 |       "redirect_url" => item[:redirect_url],
605 |       "error_reason" => item[:error_reason],
606 |       "target_port_list" => item[:target_port_list],
607 |       "target_status_code_list" => item[:target_status_code_list],
608 |       "classification" => item[:classification],
609 |       "classification_reason" => item[:classification_reason],
610 |       "conn_trace_id" => item[:conn_trace_id]
611 |     }
612 |   end
613 | end
614 | 


--------------------------------------------------------------------------------
/test/helper.rb:
--------------------------------------------------------------------------------
 1 | require 'bundler/setup'
 2 | require 'test/unit'
 3 | require 'webmock/test_unit'
 4 | require 'simplecov'
 5 | 
 6 | SimpleCov.start do
 7 |   add_filter '/test/'
 8 | end
 9 | 
10 | $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
11 | $LOAD_PATH.unshift(__dir__)
12 | require 'fluent/test'
13 | require 'fluent/test/helpers'
14 | require 'fluent/test/driver/input'
15 | require 'fluent/plugin/in_elb_log'
16 | 
17 | class Test::Unit::TestCase
18 |   include Fluent::Test::Helpers
19 |   extend Fluent::Test::Helpers
20 | end
21 | 


--------------------------------------------------------------------------------
/test/plugin/in_elb_log.rb:
--------------------------------------------------------------------------------
  1 | require_relative '../helper'
  2 | 
  3 | class Elb_LogInputTest < Test::Unit::TestCase
  4 | 
  5 |   def setup
  6 |     Fluent::Test.setup
  7 |   end
  8 | 
  9 |   DEFAULT_CONFIG = {
 10 |     access_key_id: 'dummy_access_key_id',
 11 |     secret_access_key: 'dummy_secret_access_key',
 12 |     s3_endpoint: 's3.ap-northeast-1.amazonaws.com',
 13 |     s3_bucketname: 'dummy_bucket',
 14 |     s3_prefix: 'test',
 15 |     region: 'ap-northeast-1',
 16 |     timestamp_file: 'elb_last_at.dat',
 17 |     refresh_interval: 300
 18 |   }
 19 | 
 20 |   def parse_config(conf = {})
 21 |     ''.tap{|s| conf.each { |k, v| s << "#{k} #{v}\n" } }
 22 |   end
 23 | 
 24 |   def create_driver(conf = DEFAULT_CONFIG)
 25 |     Fluent::Test::Driver::Input.new(Fluent::Plugin::Elb_LogInput).configure(parse_config conf)
 26 |   end
 27 | 
 28 |   def iam_info_url
 29 |     'http://169.254.169.254/latest/meta-data/iam/security-credentials/'
 30 |   end
 31 | 
 32 |   def use_iam_role
 33 |     stub_request(:get, iam_info_url)
 34 |       .to_return(status: [200, 'OK'], body: "hostname")
 35 |     stub_request(:get, "#{iam_info_url}hostname")
 36 |       .to_return(status: [200, 'OK'],
 37 |                  body: {
 38 |                    "AccessKeyId" => "dummy",
 39 |                    "SecretAccessKey" => "secret",
 40 |                    "Token" => "token"
 41 |                  }.to_json)
 42 |   end
 43 | 
 44 |   def iam_info_timeout
 45 |     stub_request(:get, iam_info_url).to_timeout
 46 |   end
 47 | 
 48 |   def not_use_iam_role
 49 |     stub_request(:get, iam_info_url)
 50 |       .to_return(status: [404, 'Not Found'])
 51 |   end
 52 | 
 53 |   def s3bucket_ok
 54 |     stub_request(:get, 'https://s3.ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
 55 |       .to_return(status: 200, body: "", headers: {})
 56 |     stub_request(:get, 'https://s3-ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
 57 |       .to_return(status: 200, body: "", headers: {})
 58 |   end
 59 | 
 60 |   def s3bucket_not_found
 61 |     stub_request(:get, 'https://s3.ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
 62 |       .to_return(status: 404, body: "", headers: {})
 63 |     stub_request(:get, 'https://s3-ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test')
 64 |       .to_return(status: 404, body: "", headers: {})
 65 |   end
 66 | 
 67 |   def test_configure_default
 68 |     s3bucket_ok
 69 |     use_iam_role
 70 |     assert_nothing_raised { create_driver }
 71 | 
 72 |     exception = assert_raise(Fluent::ConfigError) {
 73 |       conf = DEFAULT_CONFIG.clone
 74 |       conf.delete(:s3_bucketname)
 75 |       create_driver(conf)
 76 |     }
 77 |     assert_equal('s3_bucketname is required', exception.message)
 78 | 
 79 |     exception = assert_raise(Fluent::ConfigError) {
 80 |       conf = DEFAULT_CONFIG.clone
 81 |       conf.delete(:timestamp_file)
 82 |       create_driver(conf)
 83 |     }
 84 |     assert_equal('timestamp_file is required', exception.message)
 85 |   end
 86 | 
 87 |   def test_configure_in_EC2_with_IAM_role
 88 |     s3bucket_ok
 89 |     use_iam_role
 90 |     conf = DEFAULT_CONFIG.clone
 91 |     conf.delete(:access_key_id)
 92 |     conf.delete(:secret_access_key)
 93 |     assert_nothing_raised { create_driver(conf) }
 94 |   end
 95 | 
 96 |   def test_configure_in_EC2_without_IAM_role
 97 |     ENV['AWS_PROFILE'] = ''
 98 |     exception = assert_raise(Fluent::ConfigError) {
 99 |       s3bucket_ok
100 |       not_use_iam_role
101 |       conf = DEFAULT_CONFIG.clone
102 |       conf.delete(:access_key_id)
103 |       create_driver(conf)
104 |     }
105 |     assert_equal('access_key_id is required', exception.message)
106 | 
107 |     exception = assert_raise(Fluent::ConfigError) {
108 |       conf = DEFAULT_CONFIG.clone
109 |       conf.delete(:secret_access_key)
110 |       create_driver(conf)
111 |     }
112 |     assert_equal('secret_access_key is required', exception.message)
113 |   end
114 | 
115 |   def test_configure_outside_EC2
116 |     s3bucket_ok
117 |     iam_info_timeout
118 | 
119 |     assert_nothing_raised { create_driver }
120 |     exception = assert_raise(Fluent::ConfigError) {
121 |       conf = DEFAULT_CONFIG.clone
122 |       conf.delete(:access_key_id)
123 |       create_driver(conf)
124 |     }
125 |     assert_equal('access_key_id is required', exception.message)
126 | 
127 |     exception = assert_raise(Fluent::ConfigError) {
128 |       conf = DEFAULT_CONFIG.clone
129 |       conf.delete(:secret_access_key)
130 |       create_driver(conf)
131 |     }
132 |     assert_equal('secret_access_key is required', exception.message)
133 |   end
134 | 
135 |   def test_not_found_s3bucket
136 |     e = assert_raise(Fluent::ConfigError) {
137 |       use_iam_role
138 |       s3bucket_not_found
139 |       create_driver(DEFAULT_CONFIG.clone)
140 |     }
141 |     assert_equal('s3 bucket not found dummy_bucket', e.message)
142 |   end
143 | 
144 |   def test_logfilename_classic_lb_parse
145 |     logfile_classic = 'classic/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2017/05/03/123456789012_elasticloadbalancing_ap-northeast-1_elbname_20170503T1250Z_10.0.0.1_43nzjpdj.log'
146 | 
147 |     m = Fluent::Plugin::Elb_LogInput::LOGFILE_REGEXP.match(logfile_classic)
148 |     assert_equal('classic', m[:prefix])
149 |     assert_equal('123456789012', m[:account_id])
150 |     assert_equal('ap-northeast-1', m[:region])
151 |     assert_equal('2017/05/03', m[:logfile_date])
152 |     assert_equal('elbname', m[:logfile_elb_name])
153 |     assert_equal('20170503T1250Z', m[:elb_timestamp])
154 |     assert_equal('10.0.0.1', m[:elb_ip_address])
155 |     assert_equal('43nzjpdj', m[:logfile_hash])
156 |   end
157 | 
158 |   def test_logfilename_appication_lb_parse
159 |     logfile_applb = 'applb/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2017/05/03/123456789012_elasticloadbalancing_ap-northeast-1_app.elbname.59bfa19e900030c2_20170503T1310Z_10.0.0.1_2tko12gv.log.gz'
160 | 
161 |     m = Fluent::Plugin::Elb_LogInput::LOGFILE_REGEXP.match(logfile_applb)
162 |     assert_equal('applb', m[:prefix])
163 |     assert_equal('123456789012', m[:account_id])
164 |     assert_equal('ap-northeast-1', m[:region])
165 |     assert_equal('2017/05/03', m[:logfile_date])
166 |     assert_equal('app.elbname.59bfa19e900030c2', m[:logfile_elb_name])
167 |     assert_equal('20170503T1310Z', m[:logfile_timestamp])
168 |     assert_equal('10.0.0.1', m[:elb_ip_address])
169 |     assert_equal('2tko12gv', m[:logfile_hash])
170 |   end
171 | 
172 |   def test_log_classic_lb_parse
173 |     log = '2017-05-05T12:53:50.128456Z elbname 10.11.12.13:37852 192.168.30.186:443 0.00004 0.085372 0.000039 301 301 0 0 "GET https://elbname-123456789.ap-northeast-1.elb.amazonaws.com:443/ HTTP/1.1" "curl/7.51.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2'
174 | 
175 |     m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log)
176 |     assert_equal('2017-05-05T12:53:50.128456Z', m[:time])
177 |     assert_equal('elbname', m[:elb])
178 |     assert_equal('10.11.12.13', m[:client])
179 |     assert_equal('37852', m[:client_port])
180 |     assert_equal('192.168.30.186', m[:target])
181 |     assert_equal('443', m[:target_port])
182 |     assert_equal('0.00004', m[:request_processing_time])
183 |     assert_equal('0.085372', m[:target_processing_time])
184 |     assert_equal('0.000039', m[:response_processing_time])
185 |     assert_equal('301', m[:elb_status_code])
186 |     assert_equal('301', m[:target_status_code])
187 |     assert_equal('0', m[:received_bytes])
188 |     assert_equal('0', m[:sent_bytes])
189 |     assert_equal('GET', m[:request_method])
190 |     assert_equal('https://elbname-123456789.ap-northeast-1.elb.amazonaws.com:443/', m[:request_uri])
191 |     assert_equal('HTTP/1.1', m[:request_protocol])
192 |     assert_equal('curl/7.51.0', m[:user_agent])
193 |     assert_equal('ECDHE-RSA-AES128-GCM-SHA256', m[:ssl_cipher])
194 |     assert_equal('TLSv1.2', m[:ssl_protocol])
195 |     assert_equal(nil, m[:type])
196 |     assert_equal(nil, m[:target_group_arn])
197 |     assert_equal(nil, m[:trace_id])
198 |     assert_equal(nil, m[:conn_trace_id])
199 |   end
200 | 
201 |   def test_log_application_lb_parse
202 |     log = 'https 2017-05-05T13:07:53.468529Z app/elbname/59bfa19e900030c2 10.20.30.40:52730 192.168.30.186:443 0.006 0.000 0.086 301 301 117 507 "GET https://elbname-1121128512.ap-northeast-1.elb.amazonaws.com:443/ HTTP/1.1" "curl/7.51.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d "Root=1-590c7929-4eb4cb393d46a01d22db8473"'
203 | 
204 |     m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log)
205 |     assert_equal('2017-05-05T13:07:53.468529Z', m[:time])
206 |     assert_equal('app/elbname/59bfa19e900030c2', m[:elb])
207 |     assert_equal('10.20.30.40', m[:client])
208 |     assert_equal('52730', m[:client_port])
209 |     assert_equal('192.168.30.186', m[:target])
210 |     assert_equal('443', m[:target_port])
211 |     assert_equal('0.006', m[:request_processing_time])
212 |     assert_equal('0.000', m[:target_processing_time])
213 |     assert_equal('0.086', m[:response_processing_time])
214 |     assert_equal('301', m[:elb_status_code])
215 |     assert_equal('301', m[:target_status_code])
216 |     assert_equal('117', m[:received_bytes])
217 |     assert_equal('507', m[:sent_bytes])
218 |     assert_equal('GET', m[:request_method])
219 |     assert_equal('https://elbname-1121128512.ap-northeast-1.elb.amazonaws.com:443/', m[:request_uri])
220 |     assert_equal('HTTP/1.1', m[:request_protocol])
221 |     assert_equal('curl/7.51.0', m[:user_agent])
222 |     assert_equal('ECDHE-RSA-AES128-GCM-SHA256', m[:ssl_cipher])
223 |     assert_equal('TLSv1.2', m[:ssl_protocol])
224 |     assert_equal('https', m[:type])
225 |     assert_equal('arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d', m[:target_group_arn])
226 |     assert_equal('"Root=1-590c7929-4eb4cb393d46a01d22db8473"', m[:trace_id])
227 |     assert_equal(nil, m[:conn_trace_id])
228 |   end
229 | 
230 |   def test_grp_and_trace_fileld
231 |     log = 'http 2018-03-15T03:50:00.337397Z app/elbname/aabbccdd9988 10.248.9.92:54000 10.248.9.77:80 0.001 0.002 0.003 200 200 686 6476 "GET http://services-lb.nottherealsite.net:80/svc/example HTTP/1.1" "-" - - arn:aws:elasticloadbalancing:us-east-1:123456789123:targetgroup/example-service/1234abcd1234abcd "Root=1-xxxxxxxx-yyyyyyyyyyyyyyyyyyyzzzzz" "-" "-" 3'
232 |     m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log)
233 |     assert_equal('http', m[:type])
234 |     assert_equal('2018-03-15T03:50:00.337397Z', m[:time])
235 |     assert_equal('app/elbname/aabbccdd9988', m[:elb])
236 |     assert_equal('10.248.9.92', m[:client])
237 |     assert_equal('54000', m[:client_port])
238 |     assert_equal('10.248.9.77', m[:target])
239 |     assert_equal('80', m[:target_port])
240 |     assert_equal('0.001', m[:request_processing_time])
241 |     assert_equal('0.002', m[:target_processing_time])
242 |     assert_equal('0.003', m[:response_processing_time])
243 |     assert_equal('200', m[:elb_status_code])
244 |     assert_equal('200', m[:target_status_code])
245 |     assert_equal('686', m[:received_bytes])
246 |     assert_equal('6476', m[:sent_bytes])
247 |     assert_equal('GET', m[:request_method])
248 |     assert_equal('http://services-lb.nottherealsite.net:80/svc/example', m[:request_uri])
249 |     assert_equal('HTTP/1.1', m[:request_protocol])
250 |     assert_equal('-', m[:user_agent])
251 |     assert_equal('-', m[:ssl_cipher])
252 |     assert_equal('-', m[:ssl_protocol])
253 |     assert_equal('arn:aws:elasticloadbalancing:us-east-1:123456789123:targetgroup/example-service/1234abcd1234abcd', m[:target_group_arn])
254 |     assert_equal('"Root=1-xxxxxxxx-yyyyyyyyyyyyyyyyyyyzzzzz"', m[:trace_id])
255 |     assert_equal('domain_name', m[:domain_name])
256 |     assert_equal('chosen_cert_arn', m[:chosen_cert_arn])
257 |     assert_equal('matched_rule_priority', m[:matched_rule_priority])
258 |   end
259 | 
260 |   def test_alb_all_field
261 |     log = 'http 2019-10-26T06:10:03.157333Z app/my-alb/520e61ffffffffff 60.11.22.33:51306 192.168.30.111:443 0.010 0.097 0.001 301 301 414 507 "GET http://www.example.com:80/ HTTP/1.1" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36" ssl1 ssl2 arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4ffffffff "Root=1-123abcde-d03aafc8497211546b64c54c" "domainname" "certarn" 50000 2019-10-26T06:10:03.050000Z "forward" "redirect://url-something.com/" "error_reason" "192.168.30.186:443" "301"'
262 |     m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log)
263 |     assert_equal('http', m[:type])
264 |     assert_equal('2019-10-26T06:10:03.157333Z', m[:time])
265 | 
266 |     assert_equal('app/my-alb/520e61ffffffffff', m[:elb])
267 |     assert_equal('60.11.22.33', m[:client])
268 |     assert_equal('51306', m[:client_port])
269 |     assert_equal('192.168.30.111', m[:target])
270 |     assert_equal('443', m[:target_port])
271 | 
272 |     assert_equal('0.010', m[:request_processing_time])
273 |     assert_equal('0.097', m[:target_processing_time])
274 |     assert_equal('0.001', m[:response_processing_time])
275 |     assert_equal('301', m[:elb_status_code])
276 |     assert_equal('301', m[:target_status_code])
277 |     assert_equal('414', m[:received_bytes])
278 |     assert_equal('507', m[:sent_bytes])
279 |     assert_equal('GET', m[:request_method])
280 |     assert_equal('http://www.example.com:80/', m[:request_uri])
281 |     assert_equal('HTTP/1.1', m[:request_protocol])
282 |     assert_equal('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36', m[:user_agent])
283 |     assert_equal('ssl1', m[:ssl_cipher])
284 |     assert_equal('ssl2', m[:ssl_protocol])
285 |     assert_equal('arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4ffffffff', m[:target_group_arn])
286 |     assert_equal('"Root=1-123abcde-d03aafc8497211546b64c54c"', m[:trace_id])
287 |     assert_equal('domainname', m[:domain_name])
288 |     assert_equal('certarn', m[:chosen_cert_arn])
289 |     assert_equal('50000', m[:matched_rule_priority])
290 |     assert_equal('2019-10-26T06:10:03.050000Z', m[:request_creation_time])
291 |     assert_equal('forward', m[:actions_executed])
292 |     assert_equal('redirect://url-something.com/', m[:redirect_url])
293 |     assert_equal('error_reason', m[:error_reason])
294 |     assert_equal('"192.168.30.186:443"', m[:target_port_list])
295 |     assert_equal('"301"', m[:target_status_code_list])
296 |     assert_equal(nil, m[:classification])
297 |   end
298 | end
299 | 


--------------------------------------------------------------------------------