├── .coveralls.yml ├── .gitignore ├── .rspec ├── .travis.yml ├── CHANGELOG.md ├── Gemfile ├── Gemfile.fluentd.v0.10 ├── Gemfile.fluentd.v0.12 ├── LICENSE ├── README.md ├── Rakefile ├── fluent-plugin-grep.gemspec ├── lib └── fluent │ └── plugin │ └── out_grep.rb └── test ├── bench_out_grep.rb ├── helper.rb └── test_out_grep.rb /.coveralls.yml: -------------------------------------------------------------------------------- 1 | repo_token: iomWCR9hV8xciCfpsPydDgckjaYH4dmw9 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /*.gem 2 | ~* 3 | #* 4 | *~ 5 | .bundle 6 | *.lock 7 | .rbenv-version 8 | vendor 9 | doc/* 10 | tmp/* 11 | coverage 12 | .yardoc 13 | .ruby-version 14 | pkg/* 15 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --colour 2 | --format Fuubar 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | rvm: 2 | - 2.1.* 3 | - 2.2.* 4 | - 2.3.0 5 | gemfile: 6 | - Gemfile 7 | - Gemfile.fluentd.v0.12 8 | - Gemfile.fluentd.v0.10 9 | before_install: 10 | - gem update bundler 11 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.3.4 (2015/05/10) 2 | 3 | Fixes: 4 | 5 | * Fix `replace_invalid_sequence` was always effective 6 | 7 | Enhancements: 8 | 9 | * Some log enhancements 10 | * Support label feature of Fluentd v0.12 11 | 12 | ## 0.3.3 (2014/04/12) 13 | 14 | Enhancements: 15 | 16 | * Add `add_tag_suffix` and `remove_tag_suffix` options 17 | * Allow . in tag options to support compatibility with HandleTagNameMixin 18 | 19 | ## 0.3.2 (2014/02/04) 20 | 21 | Enhancement: 22 | 23 | - Support `log_level` option of Fleuntd v0.10.43 24 | 25 | ## 0.3.1 (2013/12/16) 26 | 27 | Changes: 28 | 29 | - Make it possible that regexp to contain a heading space on `regexpN` and `excludeN` option. 30 | 31 | ## 0.3.0 (2013/12/15) 32 | 33 | Features: 34 | 35 | - Add `regexpN` and `excludeN` option. `input_key`, `regexp`, and `exclude` options are now obsolete. 36 | 37 | ## 0.2.1 (2013/12/12) 38 | 39 | Features: 40 | 41 | - Allow to use `remove_tag_prefix` option alone 42 | 43 | ## 0.2.0 (2013/11/30) 44 | 45 | Features: 46 | 47 | - Add `remove_tag_prefix` option 48 | 49 | ## 0.1.1 (2013/11/02) 50 | 51 | Changes: 52 | 53 | - Revert String#scrub because `string-scrub` gem is only for >= ruby 2.0. 54 | 55 | ## 0.1.0 (2013/11/02) 56 | 57 | Changes: 58 | 59 | - Use String#scrub 60 | 61 | ## 0.0.3 (2013/05/14) 62 | 63 | Features: 64 | 65 | - Support to grep non-string jsonable values (such as integer, array) by #to_s. 66 | 67 | Changes: 68 | 69 | - Default tag prefix from `grep` to `greped`. 70 | 71 | ## 0.0.2 (2013/05/02) 72 | 73 | Features: 74 | 75 | - Add `replace_invalid_sequence` option 76 | 77 | ## 0.0.1 (2013/05/02) 78 | 79 | First version 80 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /Gemfile.fluentd.v0.10: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | gem 'fluentd', '~> 0.10.43' 4 | gemspec 5 | -------------------------------------------------------------------------------- /Gemfile.fluentd.v0.12: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | gem 'fluentd', '~> 0.12.0' 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Naotoshi SEO 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **NOTE: [grep filter](http://docs.fluentd.org/articles/filter_grep) is now a built-in plugin. Use the built-in plugin instead of installing this plugin.** 2 | 3 | **Or, you may find https://github.com/sonots/fluent-plugin-filter_where is more useful.** 4 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "bundler/gem_tasks" 3 | 4 | require 'rake/testtask' 5 | desc 'Run test_unit based test' 6 | Rake::TestTask.new(:test) do |t| 7 | t.libs << "test" 8 | t.test_files = Dir["test/**/test_*.rb"].sort 9 | t.verbose = true 10 | #t.warning = true 11 | end 12 | task :default => :test 13 | 14 | desc 'Open an irb session preloaded with the gem library' 15 | task :console do 16 | sh 'irb -rubygems -I lib' 17 | end 18 | task :c => :console 19 | -------------------------------------------------------------------------------- /fluent-plugin-grep.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | 4 | Gem::Specification.new do |s| 5 | s.name = "fluent-plugin-grep" 6 | s.version = "0.3.4" 7 | s.authors = ["Naotoshi Seo"] 8 | s.email = ["sonots@gmail.com"] 9 | s.homepage = "https://github.com/sonots/fluent-plugin-grep" 10 | s.summary = "fluentd plugin to grep messages" 11 | s.description = s.summary 12 | s.licenses = ["MIT"] 13 | 14 | s.rubyforge_project = "fluent-plugin-grep" 15 | 16 | s.files = `git ls-files`.split("\n") 17 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 18 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 19 | s.require_paths = ["lib"] 20 | 21 | s.add_runtime_dependency "fluentd" 22 | s.add_runtime_dependency "string-scrub" if RUBY_VERSION.to_f < 2.1 23 | s.add_development_dependency "rake" 24 | s.add_development_dependency "test-unit" 25 | s.add_development_dependency "pry" 26 | s.add_development_dependency "pry-nav" 27 | end 28 | -------------------------------------------------------------------------------- /lib/fluent/plugin/out_grep.rb: -------------------------------------------------------------------------------- 1 | class Fluent::GrepOutput < Fluent::Output 2 | Fluent::Plugin.register_output('grep', self) 3 | 4 | REGEXP_MAX_NUM = 20 5 | 6 | config_param :input_key, :string, :default => nil # obsolete 7 | config_param :regexp, :string, :default => nil # obsolete 8 | config_param :exclude, :string, :default => nil # obsolete 9 | config_param :tag, :string, :default => nil 10 | config_param :add_tag_prefix, :string, :default => nil 11 | config_param :remove_tag_prefix, :string, :default => nil 12 | config_param :add_tag_suffix, :string, :default => nil 13 | config_param :remove_tag_suffix, :string, :default => nil 14 | config_param :replace_invalid_sequence, :bool, :default => true 15 | (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}", :string, :default => nil } 16 | (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, :default => nil } 17 | 18 | # for test 19 | attr_reader :regexps 20 | attr_reader :excludes 21 | 22 | # To support log_level option implemented by Fluentd v0.10.43 23 | unless method_defined?(:log) 24 | define_method("log") { $log } 25 | end 26 | 27 | # Define `router` method of v0.12 to support v0.10 or earlier 28 | unless method_defined?(:router) 29 | define_method("router") { Fluent::Engine } 30 | end 31 | 32 | def initialize 33 | require 'string/scrub' if RUBY_VERSION.to_f < 2.1 34 | super 35 | end 36 | 37 | def configure(conf) 38 | super 39 | 40 | @regexps = {} 41 | @regexps[@input_key] = Regexp.compile(@regexp) if @input_key and @regexp 42 | (1..REGEXP_MAX_NUM).each do |i| 43 | next unless conf["regexp#{i}"] 44 | key, regexp = conf["regexp#{i}"].split(/ /, 2) 45 | raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp 46 | raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key] 47 | @regexps[key] = Regexp.compile(regexp) 48 | end 49 | 50 | @excludes = {} 51 | @excludes[@input_key] = Regexp.compile(@exclude) if @input_key and @exclude 52 | (1..REGEXP_MAX_NUM).each do |i| 53 | next unless conf["exclude#{i}"] 54 | key, exclude = conf["exclude#{i}"].split(/ /, 2) 55 | raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude 56 | raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key] 57 | @excludes[key] = Regexp.compile(exclude) 58 | end 59 | 60 | if conf['@label'].nil? and @tag.nil? and @add_tag_prefix.nil? and @remove_tag_prefix.nil? and @add_tag_suffix.nil? and @remove_tag_suffix.nil? 61 | @add_tag_prefix = 'greped' # not ConfigError to support lower version compatibility 62 | end 63 | @tag_proc = tag_proc 64 | end 65 | 66 | def emit(tag, es, chain) 67 | emit_tag = @tag_proc.call(tag) 68 | 69 | es.each do |time,record| 70 | catch(:break_loop) do 71 | @regexps.each do |key, regexp| 72 | throw :break_loop unless match(regexp, record[key].to_s) 73 | end 74 | @excludes.each do |key, exclude| 75 | throw :break_loop if match(exclude, record[key].to_s) 76 | end 77 | router.emit(emit_tag, time, record) 78 | end 79 | end 80 | 81 | chain.next 82 | rescue => e 83 | log.warn "out_grep: #{e.class} #{e.message} #{e.backtrace.first}" 84 | end 85 | 86 | private 87 | 88 | def tag_proc 89 | rstrip = Proc.new {|str, substr| str.chomp(substr) } 90 | lstrip = Proc.new {|str, substr| str.start_with?(substr) ? str[substr.size..-1] : str } 91 | tag_prefix = "#{rstrip.call(@add_tag_prefix, '.')}." if @add_tag_prefix 92 | tag_suffix = ".#{lstrip.call(@add_tag_suffix, '.')}" if @add_tag_suffix 93 | tag_prefix_match = "#{rstrip.call(@remove_tag_prefix, '.')}." if @remove_tag_prefix 94 | tag_suffix_match = ".#{lstrip.call(@remove_tag_suffix, '.')}" if @remove_tag_suffix 95 | tag_fixed = @tag if @tag 96 | if tag_fixed 97 | Proc.new {|tag| tag_fixed } 98 | elsif tag_prefix_match and tag_suffix_match 99 | Proc.new {|tag| "#{tag_prefix}#{rstrip.call(lstrip.call(tag, tag_prefix_match), tag_suffix_match)}#{tag_suffix}" } 100 | elsif tag_prefix_match 101 | Proc.new {|tag| "#{tag_prefix}#{lstrip.call(tag, tag_prefix_match)}#{tag_suffix}" } 102 | elsif tag_suffix_match 103 | Proc.new {|tag| "#{tag_prefix}#{rstrip.call(tag, tag_suffix_match)}#{tag_suffix}" } 104 | else 105 | Proc.new {|tag| "#{tag_prefix}#{tag}#{tag_suffix}" } 106 | end 107 | end 108 | 109 | def match(regexp, string) 110 | begin 111 | return regexp.match(string) 112 | rescue ArgumentError => e 113 | raise e unless @replace_invalid_sequence 114 | raise e unless e.message.index("invalid byte sequence in") == 0 115 | log.info "out_grep: invalid byte sequence is replaced in `#{string}`" 116 | string = string.scrub('?') 117 | retry 118 | end 119 | return true 120 | end 121 | end 122 | -------------------------------------------------------------------------------- /test/bench_out_grep.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helper' 2 | require 'fluent/plugin/out_grep' 3 | 4 | # setup 5 | Fluent::Test.setup 6 | config = %[ 7 | input_key message 8 | grep INFO 9 | exlude something 10 | remove_tag_prefix foo 11 | add_tag_prefix bar 12 | ] 13 | time = Time.now.to_i 14 | tag = 'foo.bar' 15 | driver = Fluent::Test::OutputTestDriver.new(Fluent::GrepOutput, tag).configure(config) 16 | 17 | # bench 18 | require 'benchmark' 19 | message = "2013/01/13T07:02:11.124202 INFO GET /ping" 20 | n = 100000 21 | Benchmark.bm(7) do |x| 22 | x.report { driver.run { n.times { driver.emit({'message' => message}, time) } } } 23 | end 24 | 25 | # BEFORE TAG_PROC 26 | # user system total real 27 | # 2.560000 0.030000 2.590000 ( 3.169847) 28 | # AFTER TAG_PROC (0.2.1) 29 | # user system total real 30 | # 2.480000 0.040000 2.520000 ( 3.085798) 31 | # AFTER regexps, exludes (0.3.0) 32 | # user system total real 33 | # 2.700000 0.050000 2.750000 ( 3.340524) 34 | # AFTER add_tag_suffix, remove_tag_suffix (0.3.3) 35 | # user system total real 36 | # 2.470000 0.020000 2.490000 ( 3.012241) 37 | -------------------------------------------------------------------------------- /test/helper.rb: -------------------------------------------------------------------------------- 1 | require 'test/unit' 2 | require 'fluent/log' 3 | require 'fluent/test' 4 | require 'fluent/version' 5 | 6 | unless defined?(Test::Unit::AssertionFailedError) 7 | class Test::Unit::AssertionFailedError < StandardError 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /test/test_out_grep.rb: -------------------------------------------------------------------------------- 1 | require_relative 'helper' 2 | require 'fluent/test' 3 | require 'fluent/plugin/out_grep' 4 | 5 | Fluent::Test.setup 6 | 7 | class GrepOutputTest < Test::Unit::TestCase 8 | setup do 9 | @tag = 'syslog.host1' 10 | @time = Fluent::Engine.now 11 | end 12 | 13 | def create_driver(conf, use_v1_config = true) 14 | Fluent::Test::OutputTestDriver.new(Fluent::GrepOutput, @tag).configure(conf, use_v1_config) 15 | end 16 | 17 | def emit(config, msgs = ['']) 18 | d = create_driver(config) 19 | d.run do 20 | msgs.each do |msg| 21 | d.emit({'foo' => 'bar', 'message' => msg}, @time) 22 | end 23 | end 24 | 25 | @instance = d.instance 26 | d.emits 27 | end 28 | 29 | sub_test_case 'configure' do 30 | test 'check default' do 31 | d = nil 32 | assert_nothing_raised do 33 | d = create_driver('') 34 | end 35 | assert_empty(d.instance.regexps) 36 | assert_empty(d.instance.excludes) 37 | end 38 | 39 | test "regexpN can contain a space" do 40 | d = create_driver(%[regexp1 message foo]) 41 | assert_equal(Regexp.compile(/ foo/), d.instance.regexps['message']) 42 | end 43 | 44 | test "excludeN can contain a space" do 45 | d = create_driver(%[exclude1 message foo]) 46 | assert_equal(Regexp.compile(/ foo/), d.instance.excludes['message']) 47 | end 48 | 49 | test "regexp contains a duplicated key" do 50 | config = %[ 51 | input_key message 52 | regexp foo 53 | regexp1 message foo 54 | ] 55 | assert_raise(Fluent::ConfigError) do 56 | create_driver(config) 57 | end 58 | end 59 | 60 | test "exclude contains a duplicated key" do 61 | config = %[ 62 | input_key message 63 | exclude foo 64 | exclude1 message foo 65 | ] 66 | assert_raise(Fluent::ConfigError) do 67 | create_driver(config) 68 | end 69 | end 70 | 71 | if Fluent::VERSION >= "0.12" 72 | test "@label" do 73 | Fluent::Engine.root_agent.add_label('@foo') 74 | d = create_driver(%[@label @foo]) 75 | label = Fluent::Engine.root_agent.find_label('@foo') 76 | assert_equal(label.event_router, d.instance.router) 77 | 78 | emits = emit(%[@label @foo], ['foo']) 79 | tag, time, record = emits.first 80 | assert_equal(@tag, tag) # tag is not modified 81 | end 82 | end 83 | end 84 | 85 | sub_test_case 'emit' do 86 | def messages 87 | [ 88 | "2013/01/13T07:02:11.124202 INFO GET /ping", 89 | "2013/01/13T07:02:13.232645 WARN POST /auth", 90 | "2013/01/13T07:02:21.542145 WARN GET /favicon.ico", 91 | "2013/01/13T07:02:43.632145 WARN POST /login", 92 | ] 93 | end 94 | 95 | test 'empty config' do 96 | emits = emit('', messages) 97 | assert_equal(4, emits.size) 98 | tag, time, record = emits.first 99 | assert_equal("greped.#{@tag}", tag) 100 | assert_not_nil(record, 'foo') 101 | assert_not_nil(record, 'message') 102 | end 103 | 104 | test 'regexp' do 105 | emits = emit("input_key message\nregexp WARN", messages) 106 | assert_equal(3, emits.size) 107 | assert_block('only WARN logs') do 108 | emits.all? { |tag, time, record| 109 | !record['message'].include?('INFO') 110 | } 111 | end 112 | end 113 | 114 | test 'regexpN' do 115 | emits = emit('regexp1 message WARN', messages) 116 | assert_equal(3, emits.size) 117 | assert_block('only WARN logs') do 118 | emits.all? { |tag, time, record| 119 | !record['message'].include?('INFO') 120 | } 121 | end 122 | end 123 | 124 | test 'exclude' do 125 | emits = emit("input_key message\nexclude favicon", messages) 126 | assert_equal(3, emits.size) 127 | assert_block('remove favicon logs') do 128 | emits.all? { |tag, time, record| 129 | !record['message'].include?('favicon') 130 | } 131 | end 132 | end 133 | 134 | test 'excludeN' do 135 | emits = emit('exclude1 message favicon', messages) 136 | assert_equal(3, emits.size) 137 | assert_block('remove favicon logs') do 138 | emits.all? { |tag, time, record| 139 | !record['message'].include?('favicon') 140 | } 141 | end 142 | end 143 | 144 | test 'tag' do 145 | emits = emit('tag foo') 146 | tag, time, record = emits.first 147 | assert_equal('foo', tag) 148 | end 149 | 150 | test 'add_tag_prefix' do 151 | emits = emit('add_tag_prefix foo') 152 | tag, time, record = emits.first 153 | assert_equal("foo.#{@tag}", tag) 154 | end 155 | 156 | test 'remove_tag_prefix' do 157 | emits = emit('remove_tag_prefix syslog') 158 | tag, time, record = emits.first 159 | assert_equal("host1", tag) 160 | end 161 | 162 | test 'add_tag_suffix' do 163 | emits = emit('add_tag_suffix foo') 164 | tag, time, record = emits.first 165 | assert_equal("#{@tag}.foo", tag) 166 | end 167 | 168 | test 'remove_tag_suffix' do 169 | emits = emit('remove_tag_suffix host1') 170 | tag, time, record = emits.first 171 | assert_equal("syslog", tag) 172 | end 173 | 174 | test 'add_tag_prefix.' do 175 | emits = emit('add_tag_prefix foo.') 176 | tag, time, record = emits.first 177 | assert_equal("foo.#{@tag}", tag) 178 | end 179 | 180 | test 'remove_tag_prefix.' do 181 | emits = emit('remove_tag_prefix syslog.') 182 | tag, time, record = emits.first 183 | assert_equal("host1", tag) 184 | end 185 | 186 | test '.add_tag_suffix' do 187 | emits = emit('add_tag_suffix .foo') 188 | tag, time, record = emits.first 189 | assert_equal("#{@tag}.foo", tag) 190 | end 191 | 192 | test '.remove_tag_suffix' do 193 | emits = emit('remove_tag_suffix .host1') 194 | tag, time, record = emits.first 195 | assert_equal("syslog", tag) 196 | end 197 | 198 | test 'all tag options' do 199 | @tag = 'syslog.foo.host1' 200 | config = %[ 201 | add_tag_prefix foo 202 | remove_tag_prefix syslog 203 | add_tag_suffix foo 204 | remove_tag_suffix host1 205 | ] 206 | emits = emit(config) 207 | tag, time, record = emits.first 208 | assert_equal("foo.foo.foo", tag) 209 | end 210 | 211 | test 'with invalid sequence' do 212 | assert_nothing_raised { 213 | emit(%[regexp1 message WARN], ["\xff".force_encoding('UTF-8')]) 214 | } 215 | end 216 | end 217 | 218 | sub_test_case 'grep non-string jsonable values' do 219 | data( 220 | 'array' => ["0"], 221 | 'hash' => ["0" => "0"], 222 | 'integer' => 0, 223 | 'float' => 0.1) 224 | test "value" do |data| 225 | emits = emit('regexp1 message 0', [data]) 226 | assert_equal(1, emits.size) 227 | end 228 | 229 | test "value boolean" do 230 | emits = emit('regexp1 message true', [true]) 231 | assert_equal(1, emits.size) 232 | end 233 | end 234 | 235 | sub_test_case 'test log' do 236 | def capture_log(log) 237 | tmp = log.out 238 | log.out = StringIO.new 239 | yield log 240 | return log.out.string 241 | ensure 242 | log.out = tmp 243 | end 244 | 245 | if Fluent::VERSION >= "0.10.43" 246 | test "log_level info" do 247 | d = create_driver('log_level info') 248 | log = d.instance.log 249 | assert_equal("", capture_log(log) {|log| log.debug "foobar" }) 250 | assert_include(capture_log(log) {|log| log.info "foobar" }, "foobar") 251 | end 252 | end 253 | 254 | test "should work" do 255 | d = create_driver('') 256 | log = d.instance.log 257 | assert_include(capture_log(log) {|log| log.info "foobar" }, "foobar") 258 | end 259 | end 260 | end 261 | --------------------------------------------------------------------------------