├── Gemfile ├── .gitignore ├── Rakefile ├── test ├── helper.rb └── plugin │ └── in_elb_log.rb ├── .travis.yml ├── fluent.conf.sample ├── LICENSE.txt ├── fluent-plugin-elb-log.gemspec ├── README.md └── lib └── fluent └── plugin └── in_elb_log.rb /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in fluent-plugin-elb-log.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | .irb_history 19 | .ruby-version 20 | .vscode 21 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | require 'rake/testtask' 4 | Rake::TestTask.new(:test) do |test| 5 | test.libs << 'lib' << 'test' 6 | test.pattern = 'test/**/*.rb' 7 | test.verbose = true 8 | test.warning = false 9 | end 10 | 11 | desc 'Run tests for all' 12 | task :default => :test 13 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | require 'test/unit' 3 | require 'webmock/test_unit' 4 | require 'simplecov' 5 | 6 | SimpleCov.start do 7 | add_filter '/test/' 8 | end 9 | 10 | $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib')) 11 | $LOAD_PATH.unshift(__dir__) 12 | require 'fluent/test' 13 | require 'fluent/test/helpers' 14 | require 'fluent/test/driver/input' 15 | require 'fluent/plugin/in_elb_log' 16 | 17 | class Test::Unit::TestCase 18 | include Fluent::Test::Helpers 19 | extend Fluent::Test::Helpers 20 | end 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | env: 2 | global: 3 | - CC_TEST_REPORTER_ID=bca5d9771c392e374d213de82dde44181a12b4c8b5d5bd37b249c81ca5bf32f1 4 | - GIT_COMMITTED_AT=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then git log -1 --pretty=format:%ct; else git log -1 --skip 1 --pretty=format:%ct; fi) 5 | language: ruby 6 | 7 | rvm: 8 | - 2.1 9 | - 2.2 10 | - 2.3.3 11 | - 2.4.1 12 | branches: 13 | only: 14 | - master 15 | 16 | before_script: 17 | - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter 18 | - chmod +x ./cc-test-reporter 19 | after_script: 20 | - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT 21 | -------------------------------------------------------------------------------- /fluent.conf.sample: -------------------------------------------------------------------------------- 1 | 2 | @type elb_log 3 | region us-east-1 4 | s3_bucketname my-elblog-bucket 5 | s3_prefix prodcution/web 6 | timestamp_file /tmp/elb_last_at.dat 7 | buf_file /tmp/fluentd-elblog.tmpfile 8 | refresh_interval 30 9 | tag elb.access 10 | use_sqs true 11 | access_key_id XXXXXXXXXXXXXXXXXXXX 12 | secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 13 | 14 | 15 | 16 | @type record_transformer 17 | 18 | timestamp ${record["request_creation_time"]} 19 | logfile_name ${record["key"]} 20 | 21 | remove_keys prefix,logfile_date,logfile_elb_name,logfile_hash,logfile_timestamp,logfile_timestamp_unixtime,key,time,s3_last_modified_unixtime 22 | 23 | 24 | 25 | @type stdout 26 | 27 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 shinsaka 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /fluent-plugin-elb-log.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "fluent-plugin-elb-log" 7 | spec.version = "1.4.0" 8 | spec.authors = ["shinsaka","jazzl0ver"] 9 | spec.email = ["shinx1265@gmail.com"] 10 | spec.summary = "Amazon ELB log input plugin" 11 | spec.description = "Amazon ELB log input plugin for fluentd" 12 | spec.homepage = "https://github.com/jazzl0ver/fluent-plugin-elb-log" 13 | spec.license = "MIT" 14 | 15 | spec.files = `git ls-files -z`.split("\x0") 16 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 17 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 18 | spec.require_paths = ["lib"] 19 | 20 | spec.add_dependency "fluentd", ">= 0.14.0", "< 2" 21 | spec.add_dependency "aws-sdk-s3", "~> 1" 22 | spec.add_dependency "aws-sdk-ec2", "~> 1" 23 | 24 | spec.add_development_dependency "bundler", ">=1.17" 25 | spec.add_development_dependency "rake", "~> 12" 26 | spec.add_development_dependency "test-unit", "~> 3.2" 27 | spec.add_development_dependency "webmock", "~>3" 28 | spec.add_development_dependency "simplecov", "~>0" 29 | end 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon ELB log input plugin for fluentd 2 | 3 | [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elb-log.svg)](https://badge.fury.io/rb/fluent-plugin-elb-log) 4 | [![Build Status](https://travis-ci.org/shinsaka/fluent-plugin-elb-log.svg?branch=master)](https://travis-ci.org/shinsaka/fluent-plugin-elb-log) 5 | [![Code Climate](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log/badges/gpa.svg)](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log) 6 | [![Test Coverage](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log/badges/coverage.svg)](https://codeclimate.com/github/shinsaka/fluent-plugin-elb-log/coverage) 7 | 8 | ## Overview 9 | - Amazon Web Services ELB log input plubin for fluentd 10 | 11 | ## Requirements 12 | 13 | | fluent-plugin-elb-log | fluentd | ruby | 14 | |-----------------------|------------|--------| 15 | | >= 0.3.0 | >= v0.14.0 | >= 2.1 | 16 | | < 0.3.0 | >= v0.12.0 | >= 1.9 | 17 | 18 | ## Installation 19 | 20 | $ fluentd-gem fluent-plugin-elb-log 21 | 22 | ## AWS ELB Settings 23 | - settings see: [Elastic Load Balancing](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/enable-access-logs.html) 24 | - developer guide: [](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html) 25 | 26 | ## Different from version 0.4.x 27 | - Using version 3 of the AWS SDK for Ruby. 28 | 29 | ## Support Application Load Balancer (ver 0.4.0 or later) 30 | - Support Access Logs for Application Load Balancer 31 | - https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html 32 | - Existing ELB is called Classic Load Balancer 33 | - http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html 34 | 35 | ## When SSL certification error 36 | log: 37 | ``` 38 | SSL_connect returned=1 errno=0 state=SSLv3 read server certificate B: certificate verify failed 39 | ``` 40 | Do env setting follows: 41 | ``` 42 | SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux) 43 | ``` 44 | 45 | ## Configuration 46 | 47 | ```config 48 | 49 | @type elb_log 50 | 51 | # following attibutes are required 52 | region 53 | s3_bucketname 54 | s3_prefix 55 | timestamp_file 56 | buf_file 57 | refresh_interval 58 | tag 59 | delete 60 | include_all_message 61 | start_time 62 | exclude_pattern_logfile_elb_name 63 | use_sqs 64 | 65 | # following attibutes are required if you don't use IAM Role 66 | access_key_id 67 | secret_access_key 68 | 69 | ``` 70 | 71 | `use_sqs` automatically creates an SQS queue named `fluent-plugin-elb-log-` 72 | and sets up the `all object create event` S3 event notification for the chosen S3 bucket. Stopping fluentd deletes both autmatically. 73 | To make it work, the following IAM policy should be attached to the instance IAM role or the user: 74 | ``` 75 | { 76 | "Version": "2012-10-17", 77 | "Statement": [ 78 | { 79 | "Sid": "FluentdPermissions", 80 | "Effect": "Allow", 81 | "Action": [ 82 | "sqs:DeleteMessage", 83 | "s3:GetObject", 84 | "sqs:ReceiveMessage", 85 | "sqs:DeleteQueue", 86 | "sqs:GetQueueAttributes", 87 | "s3:ListBucket", 88 | "s3:PutBucketNotification", 89 | "sqs:CreateQueue" 90 | ], 91 | "Resource": [ 92 | "arn:aws:sqs:*:123456789012:fluent-plugin-elb-log-*", 93 | "arn:aws:s3:::alb-logs-bucket/*", 94 | "arn:aws:s3:::alb-logs-bucket" 95 | ] 96 | } 97 | ] 98 | } 99 | ``` 100 | When `use_sqs` is false: 101 | - 300 seconds is a good value for the `refresh_interval` 102 | - the plugin executes processing the whole S3 bucket (with the respect of `s3_prefix` and `start_time`/`timestamp_file`) every `refresh_interval` 103 | 104 | When `use_sqs` is true: 105 | - `refresh_interval` of 30-60 seconds should be fine. 106 | - the plugin executes processing the whole S3 bucket (with the respect of `s3_prefix` and `start_time`/`timestamp_file`) only once (at start) and then 107 | polls the SQS queue every `refresh_interval`. 108 | 109 | ### Example setting 110 | ```config 111 | 112 | @type elb_log 113 | region us-east-1 114 | s3_bucketname my-elblog-bucket 115 | s3_prefix prodcution/web 116 | timestamp_file /tmp/elb_last_at.dat 117 | buf_file /tmp/fluentd-elblog.tmpfile 118 | refresh_interval 30 119 | tag elb.access 120 | delete false 121 | include_all_message false 122 | exclude_pattern_logfile_elb_name "^app\.(uat|qa)\." 123 | start_time 2025-02-27T10:45:00 124 | use_sqs true 125 | access_key_id XXXXXXXXXXXXXXXXXXXX 126 | secret_access_key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 127 | 128 | 129 | 130 | @type record_transformer 131 | 132 | timestamp ${record["request_creation_time"]} 133 | logfile_name ${record["key"]} 134 | 135 | remove_keys prefix,logfile_date,logfile_elb_name,logfile_hash,logfile_timestamp,logfile_timestamp_unixtime,key,time,s3_last_modified_unixtime 136 | 137 | 138 | 139 | # @type stdout 140 | @type opensearch 141 | hosts node-1,node-2,node-3 142 | port 9200 143 | scheme https 144 | logstash_format true 145 | logstash_prefix alb-logs 146 | user fluentd 147 | password secret 148 | ssl_verify true 149 | ca_file /etc/fluentd/root-ca.pem 150 | flush_interval 300s 151 | 152 | ``` 153 | 154 | ### json output example 155 | ```json 156 | { 157 | "account_id":"123456789012", 158 | "region":"ap-northeast-1", 159 | "logfile_date":"2015/06/15", 160 | "logfile_elb_name":"my-elb-name", 161 | "elb_ip_address":"52.0.0.0", 162 | "logfile_hash":"12squv5w", 163 | "logfile_timestamp":"20150615T0400Z", 164 | "key":"TEST/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2015/06/15/123456789012_elasticloadbalancing_ap-northeast-1_my-elb-name_20150615T0400Z_52.68.215.138_69squv5w.log", 165 | "prefix":"TEST", 166 | "elb_timestamp_unixtime":1434340800, 167 | "time":"2015-06-15T03:47:12.728427+0000", 168 | "elb":"my-elb-name", 169 | "client":"54.1.1.1", 170 | "client_port":"43759", 171 | "target":"10.0.0.1", 172 | "target_port":"80", 173 | "request_processing_time":4.0e-05, 174 | "target_processing_time":0.105048, 175 | "response_processing_time":2.4e-05, 176 | "elb_status_code":"200", 177 | "target_status_code":"200", 178 | "received_bytes":0, 179 | "sent_bytes":4622, 180 | "request_method":"GET", 181 | "request_uri":"https://my-elb-test.example.com/", 182 | "request_protocol":"HTTP/1.1", 183 | "user_agent":"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", 184 | "ssl_cipher":"DHE-RSA-AES128-SHA", 185 | "ssl_protocol":"TLSv1.2", 186 | "type":"http", 187 | "target_group_arn": "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d", 188 | "trace_id": "\"Root=1-xxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx\"", 189 | "domain_name": "-", 190 | "chosen_cert_arn": "-", 191 | "matched_rule_priority": "0", 192 | "request_creation_time": "2099-10-26T06:10:03.050000Z", 193 | "actions_executed": "forward", 194 | "redirect_url": "-", 195 | "error_reason": "-", 196 | "target_port_list": "\"192.168.0.1:443\"", 197 | "target_status_code_list": "\"301\"", 198 | "classification": "-", 199 | "classification_reason": "-", 200 | "conn_trace_id": "TID_xxxxxxxxxxxxxxxxxxxxxxx" 201 | } 202 | ``` 203 | -------------------------------------------------------------------------------- /test/plugin/in_elb_log.rb: -------------------------------------------------------------------------------- 1 | require_relative '../helper' 2 | 3 | class Elb_LogInputTest < Test::Unit::TestCase 4 | 5 | def setup 6 | Fluent::Test.setup 7 | end 8 | 9 | DEFAULT_CONFIG = { 10 | access_key_id: 'dummy_access_key_id', 11 | secret_access_key: 'dummy_secret_access_key', 12 | s3_endpoint: 's3.ap-northeast-1.amazonaws.com', 13 | s3_bucketname: 'dummy_bucket', 14 | s3_prefix: 'test', 15 | region: 'ap-northeast-1', 16 | timestamp_file: 'elb_last_at.dat', 17 | refresh_interval: 300 18 | } 19 | 20 | def parse_config(conf = {}) 21 | ''.tap{|s| conf.each { |k, v| s << "#{k} #{v}\n" } } 22 | end 23 | 24 | def create_driver(conf = DEFAULT_CONFIG) 25 | Fluent::Test::Driver::Input.new(Fluent::Plugin::Elb_LogInput).configure(parse_config conf) 26 | end 27 | 28 | def iam_info_url 29 | 'http://169.254.169.254/latest/meta-data/iam/security-credentials/' 30 | end 31 | 32 | def use_iam_role 33 | stub_request(:get, iam_info_url) 34 | .to_return(status: [200, 'OK'], body: "hostname") 35 | stub_request(:get, "#{iam_info_url}hostname") 36 | .to_return(status: [200, 'OK'], 37 | body: { 38 | "AccessKeyId" => "dummy", 39 | "SecretAccessKey" => "secret", 40 | "Token" => "token" 41 | }.to_json) 42 | end 43 | 44 | def iam_info_timeout 45 | stub_request(:get, iam_info_url).to_timeout 46 | end 47 | 48 | def not_use_iam_role 49 | stub_request(:get, iam_info_url) 50 | .to_return(status: [404, 'Not Found']) 51 | end 52 | 53 | def s3bucket_ok 54 | stub_request(:get, 'https://s3.ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test') 55 | .to_return(status: 200, body: "", headers: {}) 56 | stub_request(:get, 'https://s3-ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test') 57 | .to_return(status: 200, body: "", headers: {}) 58 | end 59 | 60 | def s3bucket_not_found 61 | stub_request(:get, 'https://s3.ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test') 62 | .to_return(status: 404, body: "", headers: {}) 63 | stub_request(:get, 'https://s3-ap-northeast-1.amazonaws.com/dummy_bucket?encoding-type=url&max-keys=1&prefix=test') 64 | .to_return(status: 404, body: "", headers: {}) 65 | end 66 | 67 | def test_configure_default 68 | s3bucket_ok 69 | use_iam_role 70 | assert_nothing_raised { create_driver } 71 | 72 | exception = assert_raise(Fluent::ConfigError) { 73 | conf = DEFAULT_CONFIG.clone 74 | conf.delete(:s3_bucketname) 75 | create_driver(conf) 76 | } 77 | assert_equal('s3_bucketname is required', exception.message) 78 | 79 | exception = assert_raise(Fluent::ConfigError) { 80 | conf = DEFAULT_CONFIG.clone 81 | conf.delete(:timestamp_file) 82 | create_driver(conf) 83 | } 84 | assert_equal('timestamp_file is required', exception.message) 85 | end 86 | 87 | def test_configure_in_EC2_with_IAM_role 88 | s3bucket_ok 89 | use_iam_role 90 | conf = DEFAULT_CONFIG.clone 91 | conf.delete(:access_key_id) 92 | conf.delete(:secret_access_key) 93 | assert_nothing_raised { create_driver(conf) } 94 | end 95 | 96 | def test_configure_in_EC2_without_IAM_role 97 | ENV['AWS_PROFILE'] = '' 98 | exception = assert_raise(Fluent::ConfigError) { 99 | s3bucket_ok 100 | not_use_iam_role 101 | conf = DEFAULT_CONFIG.clone 102 | conf.delete(:access_key_id) 103 | create_driver(conf) 104 | } 105 | assert_equal('access_key_id is required', exception.message) 106 | 107 | exception = assert_raise(Fluent::ConfigError) { 108 | conf = DEFAULT_CONFIG.clone 109 | conf.delete(:secret_access_key) 110 | create_driver(conf) 111 | } 112 | assert_equal('secret_access_key is required', exception.message) 113 | end 114 | 115 | def test_configure_outside_EC2 116 | s3bucket_ok 117 | iam_info_timeout 118 | 119 | assert_nothing_raised { create_driver } 120 | exception = assert_raise(Fluent::ConfigError) { 121 | conf = DEFAULT_CONFIG.clone 122 | conf.delete(:access_key_id) 123 | create_driver(conf) 124 | } 125 | assert_equal('access_key_id is required', exception.message) 126 | 127 | exception = assert_raise(Fluent::ConfigError) { 128 | conf = DEFAULT_CONFIG.clone 129 | conf.delete(:secret_access_key) 130 | create_driver(conf) 131 | } 132 | assert_equal('secret_access_key is required', exception.message) 133 | end 134 | 135 | def test_not_found_s3bucket 136 | e = assert_raise(Fluent::ConfigError) { 137 | use_iam_role 138 | s3bucket_not_found 139 | create_driver(DEFAULT_CONFIG.clone) 140 | } 141 | assert_equal('s3 bucket not found dummy_bucket', e.message) 142 | end 143 | 144 | def test_logfilename_classic_lb_parse 145 | logfile_classic = 'classic/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2017/05/03/123456789012_elasticloadbalancing_ap-northeast-1_elbname_20170503T1250Z_10.0.0.1_43nzjpdj.log' 146 | 147 | m = Fluent::Plugin::Elb_LogInput::LOGFILE_REGEXP.match(logfile_classic) 148 | assert_equal('classic', m[:prefix]) 149 | assert_equal('123456789012', m[:account_id]) 150 | assert_equal('ap-northeast-1', m[:region]) 151 | assert_equal('2017/05/03', m[:logfile_date]) 152 | assert_equal('elbname', m[:logfile_elb_name]) 153 | assert_equal('20170503T1250Z', m[:elb_timestamp]) 154 | assert_equal('10.0.0.1', m[:elb_ip_address]) 155 | assert_equal('43nzjpdj', m[:logfile_hash]) 156 | end 157 | 158 | def test_logfilename_appication_lb_parse 159 | logfile_applb = 'applb/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2017/05/03/123456789012_elasticloadbalancing_ap-northeast-1_app.elbname.59bfa19e900030c2_20170503T1310Z_10.0.0.1_2tko12gv.log.gz' 160 | 161 | m = Fluent::Plugin::Elb_LogInput::LOGFILE_REGEXP.match(logfile_applb) 162 | assert_equal('applb', m[:prefix]) 163 | assert_equal('123456789012', m[:account_id]) 164 | assert_equal('ap-northeast-1', m[:region]) 165 | assert_equal('2017/05/03', m[:logfile_date]) 166 | assert_equal('app.elbname.59bfa19e900030c2', m[:logfile_elb_name]) 167 | assert_equal('20170503T1310Z', m[:logfile_timestamp]) 168 | assert_equal('10.0.0.1', m[:elb_ip_address]) 169 | assert_equal('2tko12gv', m[:logfile_hash]) 170 | end 171 | 172 | def test_log_classic_lb_parse 173 | log = '2017-05-05T12:53:50.128456Z elbname 10.11.12.13:37852 192.168.30.186:443 0.00004 0.085372 0.000039 301 301 0 0 "GET https://elbname-123456789.ap-northeast-1.elb.amazonaws.com:443/ HTTP/1.1" "curl/7.51.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2' 174 | 175 | m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log) 176 | assert_equal('2017-05-05T12:53:50.128456Z', m[:time]) 177 | assert_equal('elbname', m[:elb]) 178 | assert_equal('10.11.12.13', m[:client]) 179 | assert_equal('37852', m[:client_port]) 180 | assert_equal('192.168.30.186', m[:target]) 181 | assert_equal('443', m[:target_port]) 182 | assert_equal('0.00004', m[:request_processing_time]) 183 | assert_equal('0.085372', m[:target_processing_time]) 184 | assert_equal('0.000039', m[:response_processing_time]) 185 | assert_equal('301', m[:elb_status_code]) 186 | assert_equal('301', m[:target_status_code]) 187 | assert_equal('0', m[:received_bytes]) 188 | assert_equal('0', m[:sent_bytes]) 189 | assert_equal('GET', m[:request_method]) 190 | assert_equal('https://elbname-123456789.ap-northeast-1.elb.amazonaws.com:443/', m[:request_uri]) 191 | assert_equal('HTTP/1.1', m[:request_protocol]) 192 | assert_equal('curl/7.51.0', m[:user_agent]) 193 | assert_equal('ECDHE-RSA-AES128-GCM-SHA256', m[:ssl_cipher]) 194 | assert_equal('TLSv1.2', m[:ssl_protocol]) 195 | assert_equal(nil, m[:type]) 196 | assert_equal(nil, m[:target_group_arn]) 197 | assert_equal(nil, m[:trace_id]) 198 | assert_equal(nil, m[:conn_trace_id]) 199 | end 200 | 201 | def test_log_application_lb_parse 202 | log = 'https 2017-05-05T13:07:53.468529Z app/elbname/59bfa19e900030c2 10.20.30.40:52730 192.168.30.186:443 0.006 0.000 0.086 301 301 117 507 "GET https://elbname-1121128512.ap-northeast-1.elb.amazonaws.com:443/ HTTP/1.1" "curl/7.51.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d "Root=1-590c7929-4eb4cb393d46a01d22db8473"' 203 | 204 | m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log) 205 | assert_equal('2017-05-05T13:07:53.468529Z', m[:time]) 206 | assert_equal('app/elbname/59bfa19e900030c2', m[:elb]) 207 | assert_equal('10.20.30.40', m[:client]) 208 | assert_equal('52730', m[:client_port]) 209 | assert_equal('192.168.30.186', m[:target]) 210 | assert_equal('443', m[:target_port]) 211 | assert_equal('0.006', m[:request_processing_time]) 212 | assert_equal('0.000', m[:target_processing_time]) 213 | assert_equal('0.086', m[:response_processing_time]) 214 | assert_equal('301', m[:elb_status_code]) 215 | assert_equal('301', m[:target_status_code]) 216 | assert_equal('117', m[:received_bytes]) 217 | assert_equal('507', m[:sent_bytes]) 218 | assert_equal('GET', m[:request_method]) 219 | assert_equal('https://elbname-1121128512.ap-northeast-1.elb.amazonaws.com:443/', m[:request_uri]) 220 | assert_equal('HTTP/1.1', m[:request_protocol]) 221 | assert_equal('curl/7.51.0', m[:user_agent]) 222 | assert_equal('ECDHE-RSA-AES128-GCM-SHA256', m[:ssl_cipher]) 223 | assert_equal('TLSv1.2', m[:ssl_protocol]) 224 | assert_equal('https', m[:type]) 225 | assert_equal('arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d', m[:target_group_arn]) 226 | assert_equal('"Root=1-590c7929-4eb4cb393d46a01d22db8473"', m[:trace_id]) 227 | assert_equal(nil, m[:conn_trace_id]) 228 | end 229 | 230 | def test_grp_and_trace_fileld 231 | log = 'http 2018-03-15T03:50:00.337397Z app/elbname/aabbccdd9988 10.248.9.92:54000 10.248.9.77:80 0.001 0.002 0.003 200 200 686 6476 "GET http://services-lb.nottherealsite.net:80/svc/example HTTP/1.1" "-" - - arn:aws:elasticloadbalancing:us-east-1:123456789123:targetgroup/example-service/1234abcd1234abcd "Root=1-xxxxxxxx-yyyyyyyyyyyyyyyyyyyzzzzz" "-" "-" 3' 232 | m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log) 233 | assert_equal('http', m[:type]) 234 | assert_equal('2018-03-15T03:50:00.337397Z', m[:time]) 235 | assert_equal('app/elbname/aabbccdd9988', m[:elb]) 236 | assert_equal('10.248.9.92', m[:client]) 237 | assert_equal('54000', m[:client_port]) 238 | assert_equal('10.248.9.77', m[:target]) 239 | assert_equal('80', m[:target_port]) 240 | assert_equal('0.001', m[:request_processing_time]) 241 | assert_equal('0.002', m[:target_processing_time]) 242 | assert_equal('0.003', m[:response_processing_time]) 243 | assert_equal('200', m[:elb_status_code]) 244 | assert_equal('200', m[:target_status_code]) 245 | assert_equal('686', m[:received_bytes]) 246 | assert_equal('6476', m[:sent_bytes]) 247 | assert_equal('GET', m[:request_method]) 248 | assert_equal('http://services-lb.nottherealsite.net:80/svc/example', m[:request_uri]) 249 | assert_equal('HTTP/1.1', m[:request_protocol]) 250 | assert_equal('-', m[:user_agent]) 251 | assert_equal('-', m[:ssl_cipher]) 252 | assert_equal('-', m[:ssl_protocol]) 253 | assert_equal('arn:aws:elasticloadbalancing:us-east-1:123456789123:targetgroup/example-service/1234abcd1234abcd', m[:target_group_arn]) 254 | assert_equal('"Root=1-xxxxxxxx-yyyyyyyyyyyyyyyyyyyzzzzz"', m[:trace_id]) 255 | assert_equal('domain_name', m[:domain_name]) 256 | assert_equal('chosen_cert_arn', m[:chosen_cert_arn]) 257 | assert_equal('matched_rule_priority', m[:matched_rule_priority]) 258 | end 259 | 260 | def test_alb_all_field 261 | log = 'http 2019-10-26T06:10:03.157333Z app/my-alb/520e61ffffffffff 60.11.22.33:51306 192.168.30.111:443 0.010 0.097 0.001 301 301 414 507 "GET http://www.example.com:80/ HTTP/1.1" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36" ssl1 ssl2 arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4ffffffff "Root=1-123abcde-d03aafc8497211546b64c54c" "domainname" "certarn" 50000 2019-10-26T06:10:03.050000Z "forward" "redirect://url-something.com/" "error_reason" "192.168.30.186:443" "301"' 262 | m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log) 263 | assert_equal('http', m[:type]) 264 | assert_equal('2019-10-26T06:10:03.157333Z', m[:time]) 265 | 266 | assert_equal('app/my-alb/520e61ffffffffff', m[:elb]) 267 | assert_equal('60.11.22.33', m[:client]) 268 | assert_equal('51306', m[:client_port]) 269 | assert_equal('192.168.30.111', m[:target]) 270 | assert_equal('443', m[:target_port]) 271 | 272 | assert_equal('0.010', m[:request_processing_time]) 273 | assert_equal('0.097', m[:target_processing_time]) 274 | assert_equal('0.001', m[:response_processing_time]) 275 | assert_equal('301', m[:elb_status_code]) 276 | assert_equal('301', m[:target_status_code]) 277 | assert_equal('414', m[:received_bytes]) 278 | assert_equal('507', m[:sent_bytes]) 279 | assert_equal('GET', m[:request_method]) 280 | assert_equal('http://www.example.com:80/', m[:request_uri]) 281 | assert_equal('HTTP/1.1', m[:request_protocol]) 282 | assert_equal('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36', m[:user_agent]) 283 | assert_equal('ssl1', m[:ssl_cipher]) 284 | assert_equal('ssl2', m[:ssl_protocol]) 285 | assert_equal('arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4ffffffff', m[:target_group_arn]) 286 | assert_equal('"Root=1-123abcde-d03aafc8497211546b64c54c"', m[:trace_id]) 287 | assert_equal('domainname', m[:domain_name]) 288 | assert_equal('certarn', m[:chosen_cert_arn]) 289 | assert_equal('50000', m[:matched_rule_priority]) 290 | assert_equal('2019-10-26T06:10:03.050000Z', m[:request_creation_time]) 291 | assert_equal('forward', m[:actions_executed]) 292 | assert_equal('redirect://url-something.com/', m[:redirect_url]) 293 | assert_equal('error_reason', m[:error_reason]) 294 | assert_equal('"192.168.30.186:443"', m[:target_port_list]) 295 | assert_equal('"301"', m[:target_status_code_list]) 296 | assert_equal(nil, m[:classification]) 297 | end 298 | end 299 | -------------------------------------------------------------------------------- /lib/fluent/plugin/in_elb_log.rb: -------------------------------------------------------------------------------- 1 | require 'time' 2 | require 'zlib' 3 | require 'fileutils' 4 | require 'aws-sdk-s3' 5 | require 'aws-sdk-ec2' 6 | require 'aws-sdk-sqs' 7 | require 'fluent/input' 8 | require 'digest/sha1' 9 | 10 | class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input 11 | Fluent::Plugin.register_input('elb_log', self) 12 | 13 | helpers :timer 14 | 15 | LOGFILE_REGEXP = /^((?.+?)\/|)AWSLogs\/(?[0-9]{12})\/elasticloadbalancing\/(?.+?)\/(?[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?[^_]+)_(?[0-9]{8}T[0-9]{4}Z)_(?.+?)_(?.+)\.log(.gz)?$/ 16 | ACCESSLOG_REGEXP = /^((?[a-z0-9]+) )?(?