├── .gitignore
├── Gemfile
├── LICENSE.txt
├── README.md
├── Rakefile
├── fluent-plugin-bufferize.gemspec
├── lib
    └── fluent
    │   └── plugin
    │       └── out_bufferize.rb
└── test
    ├── helper.rb
    └── plugin
        └── test_out_bufferize.rb


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.gem
 2 | *.rbc
 3 | .bundle
 4 | .config
 5 | .yardoc
 6 | Gemfile.lock
 7 | InstalledFiles
 8 | _yardoc
 9 | coverage
10 | doc/
11 | lib/bundler/man
12 | pkg
13 | rdoc
14 | spec/reports
15 | test/tmp
16 | test/version_tmp
17 | tmp
18 | vendor/
19 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | 
3 | # Specify your gem's dependencies in fluent-plugin-bufferize.gemspec
4 | gemspec
5 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2013 Masahiro Sano
 2 | 
 3 | MIT License
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining
 6 | a copy of this software and associated documentation files (the
 7 | "Software"), to deal in the Software without restriction, including
 8 | without limitation the rights to use, copy, modify, merge, publish,
 9 | distribute, sublicense, and/or sell copies of the Software, and to
10 | permit persons to whom the Software is furnished to do so, subject to
11 | the following conditions:
12 | 
13 | The above copyright notice and this permission notice shall be
14 | included in all copies or substantial portions of the Software.
15 | 
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # fluent-plugin-bufferize, a plugin for [Fluentd](http://fluentd.org)
 2 | 
 3 | An adapter plugin which enables existing non-buffered plugins to resend messages easily in case of unexpected exceptions without creating duplicate messages.
 4 | 
 5 | ## Why
 6 | 
 7 | Buffered plugin accumulates many messages in buffer and sends all the messages at a same time. There are many APIs that does not support such reqeusts like bulk-insert. In that case, you have to use non-buffered output and implement resend meschanism yourself because non-buffered output lacks exception handling functionality. 
 8 | 
 9 | To use this plugin, you just have to create non-buffered plugin without caring exception handling. If an exception happens in your plugin, the request is issued again automatically. With file buffer, none of messages are lost even on sudden fluentd process down.
10 | 
11 | ## Configuration
12 | 
13 | Just embrace existing configuration by <config> directive.
14 | 
15 | If you use following configuration:
16 | 
17 | ```
18 | <match *>
19 |   type http
20 |   endpoint_url    http://foo.bar.com/
21 |   http_method     put
22 | </match>
23 | ```
24 | 
25 | Modify it like this:
26 | 
27 | ```
28 | <match *>
29 |   type bufferize
30 |   buffer_type file
31 |   buffer_path /var/log/fluent/myapp.*.buffer
32 |   <config>
33 |     type http
34 |     endpoint_url    http://foo.bar.com/
35 |     http_method     put
36 |   </config>
37 | </match>
38 | ```
39 | 
40 | This is a buffered output plugin. For more information about parameters, please refer [official document](http://docs.fluentd.org/articles/buffer-plugin-overview).
41 | 
42 | ## Example of application
43 | 
44 | These plugins are good compatibility to fluent-plugin-bufferize.
45 | 
46 | - [fluent-plugin-out-http](https://github.com/ento/fluent-plugin-out-http)
47 | - [fluent-plugin-jubatus](https://github.com/katsyoshi/fluent-plugin-jubatus)
48 | - [fluent-plugin-irc](https://github.com/choplin/fluent-plugin-irc)
49 | 
50 | ## Installation
51 | 
52 | Add this line to your application's Gemfile:
53 | 
54 |     gem 'fluent-plugin-bufferize'
55 | 
56 | And then execute:
57 | 
58 |     $ bundle
59 | 
60 | Or install it yourself as:
61 | 
62 |     $ gem install fluent-plugin-bufferize
63 | 
64 | ## Contributing
65 | 
66 | 1. Fork it
67 | 2. Create your feature branch (`git checkout -b my-new-feature`)
68 | 3. Commit your changes (`git commit -am 'Add some feature'`)
69 | 4. Push to the branch (`git push origin my-new-feature`)
70 | 5. Create new Pull Request
71 | 
72 | ## Copyright
73 | 
74 | <table>
75 |   <tr>
76 |     <td>Author</td><td>Masahiro Sano <sabottenda@gmail.com></td>
77 |   </tr>
78 |   <tr>
79 |     <td>Copyright</td><td>Copyright (c) 2013- Masahiro Sano</td>
80 |   </tr>
81 |   <tr>
82 |     <td>License</td><td>MIT License</td>
83 |   </tr>
84 | </table>
85 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | require "bundler"
 2 | require "bundler/gem_tasks"
 3 | 
 4 | require 'rake/testtask'
 5 | 
 6 | Rake::TestTask.new(:test) do |test|
 7 |   test.libs << 'lib' << 'test'
 8 |   test.pattern = 'test/**/test_*.rb'
 9 |   test.verbose = true
10 | end
11 | 
12 | task :default => [:test]
13 | 
14 | 


--------------------------------------------------------------------------------
/fluent-plugin-bufferize.gemspec:
--------------------------------------------------------------------------------
 1 | # coding: utf-8
 2 | lib = File.expand_path('../lib', __FILE__)
 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 4 | 
 5 | Gem::Specification.new do |spec|
 6 |   spec.name          = "fluent-plugin-bufferize"
 7 |   spec.version       = "0.0.2"
 8 |   spec.authors       = ["Masahiro Sano"]
 9 |   spec.email         = ["sabottenda@gmail.com"]
10 |   spec.description   = %q{A fluentd plugin that enhances existing non-buffered output plugin as buffered plugin.}
11 |   spec.summary       = %q{A fluentd plugin that enhances existing non-buffered output plugin as buffered plugin.}
12 |   spec.homepage      = "https://github.com/sabottenda/fluent-plugin-bufferize"
13 |   spec.license       = "MIT"
14 | 
15 |   spec.files         = `git ls-files`.split($/)
16 |   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17 |   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
18 |   spec.require_paths = ["lib"]
19 | 
20 |   spec.add_dependency "fluentd", "~> 0.14.0"
21 |   spec.add_development_dependency "bundler", "> 1.3"
22 |   spec.add_development_dependency "rake"
23 | end
24 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/out_bufferize.rb:
--------------------------------------------------------------------------------
  1 | module Fluent
  2 |   class BufferizeOutput < BufferedOutput
  3 |     Plugin.register_output('bufferize', self)
  4 | 
  5 |     class PosKeeper
  6 |       FILE_PERMISSION = 0644
  7 | 
  8 |       @@instances = {}
  9 | 
 10 |       def self.get(chunk)
 11 |         @@instances[chunk.unique_id] ||= PosKeeper.new(chunk)
 12 |         @@instances[chunk.unique_id]
 13 |       end
 14 | 
 15 |       def self.remove(chunk)
 16 |         @@instances.delete(chunk.unique_id)
 17 |       end
 18 | 
 19 |       def initialize(chunk)
 20 |         @id = chunk.unique_id
 21 |         @count = 0
 22 |         @chunk = chunk
 23 | 
 24 |         if chunk.respond_to? :path
 25 |           @path = chunk.path +  ".pos"
 26 |           mode = File::CREAT | File::RDWR
 27 |           perm = FILE_PERMISSION
 28 |           @io = File.open(@path, mode, perm)
 29 |           @io.sync = true
 30 |           line = @io.gets
 31 |           @count = line ? line.to_i : 0
 32 |           @type = :file
 33 |         else
 34 |           @type = :mem
 35 |         end
 36 |       end
 37 | 
 38 |       def each(&block)
 39 |         @chunk.open do |io|
 40 |           u = MessagePack::Unpacker.new(io)
 41 |           begin
 42 |             if @count > 0
 43 |               $log.debug "Bufferize: skip first #{@count} messages" 
 44 |               @count.times do
 45 |                 u.skip
 46 |               end
 47 |             end
 48 | 
 49 |             loop do
 50 |               tag, time, record = u.read
 51 |               yield(tag, time, record)
 52 |               increment
 53 |             end
 54 | 
 55 |           rescue EOFError
 56 |           end
 57 |         end
 58 |         remove
 59 |       end
 60 | 
 61 |       def increment
 62 |         @count += 1
 63 |         if @type == :file
 64 |           @io.seek(0, IO::SEEK_SET)
 65 |           @io.puts(@count)
 66 |         end
 67 |       end
 68 | 
 69 |       def remove
 70 |         if @type == :file
 71 |           @io.close unless @io.closed?
 72 |           File.unlink(@path)
 73 |         end
 74 |       end      
 75 |     end
 76 | 
 77 | 
 78 |     attr_reader :output
 79 | 
 80 |     def initialize
 81 |       super
 82 |     end
 83 | 
 84 |     def configure(conf)
 85 |       super
 86 | 
 87 |       configs = conf.elements.select{|e| e.name == 'config'}
 88 |       if configs.size != 1
 89 |         raise ConfigError, "Befferize: just one <config> directive is required"
 90 |       end
 91 | 
 92 |       type = configs.first['type']
 93 |       unless type
 94 |         raise ConfigError, "Befferize: 'type' parameter is required in <config> directive"
 95 |       end
 96 | 
 97 |       @output = Plugin.new_output(type)
 98 |       @output.configure(configs.first)
 99 |     end
100 | 
101 |     def start
102 |       super
103 |       @output.start
104 |     end
105 | 
106 |     def shutdown
107 |       super
108 |       @output.shutdown
109 |     end
110 | 
111 |     def format(tag, time, record)
112 |       [tag, time, record].to_msgpack
113 |     end
114 | 
115 |     def write(chunk)
116 |       PosKeeper.get(chunk).each { |tag, time, record |
117 |         @output.emit(tag, OneEventStream.new(time, record), NullOutputChain.instance)
118 |       }
119 |       PosKeeper.remove(chunk)
120 |     end
121 |   end
122 | end
123 | 


--------------------------------------------------------------------------------
/test/helper.rb:
--------------------------------------------------------------------------------
 1 | require 'rubygems'
 2 | require 'bundler'
 3 | begin
 4 |   Bundler.setup(:default, :development)
 5 | rescue Bundler::BundlerError => e
 6 |   $stderr.puts e.message
 7 |   $stderr.puts "Run `bundle install` to install missing gems"
 8 |   exit e.status_code
 9 | end
10 | require 'test/unit'
11 | 
12 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13 | $LOAD_PATH.unshift(File.dirname(__FILE__))
14 | require 'fluent/test'
15 | unless ENV.has_key?('VERBOSE')
16 |   nulllogger = Object.new
17 |   nulllogger.instance_eval {|obj|
18 |     def method_missing(method, *args)
19 |       # pass
20 |     end
21 |   }
22 |   $log = nulllogger
23 | end
24 | 
25 | require 'fluent/plugin/out_bufferize'
26 | 
27 | class Test::Unit::TestCase
28 | end
29 | 


--------------------------------------------------------------------------------
/test/plugin/test_out_bufferize.rb:
--------------------------------------------------------------------------------
  1 | require 'helper'
  2 | 
  3 | class BufferizeOutputTest < Test::Unit::TestCase
  4 |   def setup
  5 |     Fluent::Test.setup
  6 |     FileUtils.rm_rf('tmp')
  7 |     FileUtils.mkdir_p('tmp')
  8 |   end
  9 | 
 10 |   def teardown
 11 |     FileUtils.mkdir_p('tmp')
 12 |   end
 13 | 
 14 |   BASE_CONFIG = %[
 15 | type bufferize
 16 | ]
 17 |   CONFIG_NO_CONFIG = BASE_CONFIG
 18 |   CONFIG_NO_TYPE = BASE_CONFIG + %[
 19 | <config>
 20 | </config>
 21 | ]
 22 |   CONFIG_WITH_TYPE = BASE_CONFIG + %[
 23 | <config>
 24 |   type test
 25 | </config>
 26 | ]
 27 | 
 28 |   def create_driver(conf = CONFIG_WITH_TYPE, tag='test')
 29 |     Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferizeOutput, tag).configure(conf)
 30 |   end
 31 | 
 32 |   def test_configure
 33 |     assert_raise(Fluent::ConfigError) {
 34 |       create_driver(CONFIG_NO_CONFIG)
 35 |     }
 36 |     assert_raise(Fluent::ConfigError) {
 37 |       create_driver(CONFIG_NO_TYPE)
 38 |     }
 39 |     assert_nothing_raised(Fluent::ConfigError) {
 40 |       create_driver(CONFIG_WITH_TYPE)
 41 |     }
 42 |   end
 43 | 
 44 |   def create_resend_test_driver(conf = CONFIG_WITH_TYPE, tag='test')
 45 |     output = Fluent::Plugin.new_output('test')
 46 |     output.configure('name' => 'output')
 47 |     output.define_singleton_method(:start) {}
 48 |     output.define_singleton_method(:shutdown) {}
 49 |     output.define_singleton_method(:emit) do |tag, es, chain|
 50 |       @count ||= 0
 51 |       es.each do |time, record|
 52 |         @count += 1
 53 |         raise if (@count % 3) == 0
 54 |         super(tag, [[time, record]], chain)
 55 |       end
 56 |     end
 57 | 
 58 |     d = create_driver
 59 |     d.instance.instance_eval { @output = output }
 60 |     d
 61 |   end  
 62 | 
 63 |   def test_resend_with_memory_buffer
 64 |     d = create_resend_test_driver
 65 | 
 66 |     time = Time.parse("2013-11-02 12:12:12 UTC").to_i
 67 |     entries = []
 68 |     1.upto(5) { |i|
 69 |       entries << [time, {"a"=>i}]
 70 |     }
 71 | 
 72 |     es = Fluent::ArrayEventStream.new(entries)
 73 |     buffer = d.instance.format_stream('test', es)
 74 |     chunk = Fluent::MemoryBufferChunk.new('', buffer)
 75 | 
 76 |     assert_raise(RuntimeError) {
 77 |       d.instance.write(chunk)
 78 |     }
 79 |     assert_equal [
 80 |       {"a"=>1}, {"a"=>2},
 81 |     ], d.instance.output.records
 82 | 
 83 |     assert_raise(RuntimeError) {
 84 |       d.instance.write(chunk)
 85 |     }
 86 |     assert_equal [
 87 |       {"a"=>1}, {"a"=>2}, {"a"=>3}, {"a"=>4},
 88 |     ], d.instance.output.records
 89 | 
 90 |     assert_nothing_raised(RuntimeError) {
 91 |       d.instance.write(chunk)
 92 |     }
 93 |     assert_equal [
 94 |       {"a"=>1}, {"a"=>2}, {"a"=>3}, {"a"=>4}, {"a"=>5},
 95 |     ], d.instance.output.records
 96 |   end
 97 | 
 98 |   def test_resend_with_file_buffer
 99 |     d = create_resend_test_driver(CONFIG_WITH_TYPE + %[
100 | buffer_type file
101 | ])
102 | 
103 |     time = Time.parse("2013-11-02 12:12:12 UTC").to_i
104 |     entries = []
105 |     1.upto(5) { |i|
106 |       entries << [time, {"b"=>i}]
107 |     }
108 | 
109 |     es = Fluent::ArrayEventStream.new(entries)
110 |     buffer = d.instance.format_stream('test', es)
111 |     chunk = Fluent::MemoryBufferChunk.new('', buffer)
112 | 
113 |     es = Fluent::ArrayEventStream.new(entries)
114 |     chunk = Fluent::FileBufferChunk.new('', './tmp/test_buffer', 'xyz', "a+", nil)
115 |     chunk << d.instance.format_stream('test', es)
116 |     pos_file_path = "#{chunk.path}.pos"
117 | 
118 |     assert_raise(RuntimeError) {
119 |       d.instance.write(chunk)
120 |     }
121 |     assert_equal [
122 |       {"b"=>1}, {"b"=>2},
123 |     ], d.instance.output.records
124 |     assert File.exists?(pos_file_path)
125 |     assert_equal `head #{pos_file_path}`.chomp.to_i, 2
126 | 
127 |     assert_raise(RuntimeError) {
128 |       d.instance.write(chunk)
129 |     }
130 |     assert_equal [
131 |       {"b"=>1}, {"b"=>2}, {"b"=>3}, {"b"=>4},
132 |     ], d.instance.output.records
133 |     assert File.exists?(pos_file_path)
134 |     assert_equal `head #{pos_file_path}`.chomp.to_i, 4
135 | 
136 |     assert_nothing_raised(RuntimeError) {
137 |       d.instance.write(chunk)
138 |     }
139 |     assert_equal [
140 |       {"b"=>1}, {"b"=>2}, {"b"=>3}, {"b"=>4}, {"b"=>5},
141 |     ], d.instance.output.records
142 |     assert !File.exists?(pos_file_path)
143 |   end
144 | 
145 | end
146 | 


--------------------------------------------------------------------------------