├── .gitignore
├── Gemfile
├── LICENSE.txt
├── README.md
├── Rakefile
├── fluent-plugin-postgres.gemspec
├── lib
    └── fluent
    │   └── plugin
    │       └── out_postgres.rb
└── test
    ├── helper.rb
    └── plugin
        └── test_out_postgres.rb


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.gem
 2 | *.rbc
 3 | .bundle
 4 | .config
 5 | .yardoc
 6 | Gemfile.lock
 7 | InstalledFiles
 8 | _yardoc
 9 | coverage
10 | doc/
11 | lib/bundler/man
12 | pkg
13 | rdoc
14 | spec/reports
15 | test/tmp
16 | test/version_tmp
17 | tmp
18 | test.conf
19 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source :rubygems
2 | 
3 | gemspec
4 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | Copyright 2013 Uken Games
 2 | 
 3 | Licensed under the Apache License, Version 2.0 (the "License");
 4 | you may not use this file except in compliance with the License.
 5 | You may obtain a copy of the License at
 6 | 
 7 |     http://www.apache.org/licenses/LICENSE-2.0
 8 | 
 9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # fluent-plugin-postgres, a plugin for [Fluentd](http://fluentd.org)
  2 | 
  3 | ## Installation:
  4 | 
  5 | - Prereq: Install postgresql headers: `apt-get install libpq-dev`
  6 | - Install the gem:
  7 |   - `gem install fluent-plugin-postgres` or
  8 |   - `/usr/lib/fluent/ruby/bin/fluent-gem install fluent-plugin-postgres`
  9 | 
 10 | ## Changes from mysql:
 11 | 
 12 | - We currently don't support json format
 13 | - You need to specify a SQL query
 14 | - Placeholders are numbered (yeah, I know).
 15 | 
 16 | Other than that, just bear in mind that it's Postgres SQL.
 17 | 
 18 | ### Quick example
 19 | 
 20 |     <match output.by.sql.*>
 21 |       type postgres
 22 |       host master.db.service.local
 23 |       # port 3306 # default
 24 |       database application_logs
 25 |       username myuser
 26 |       password mypass
 27 |       key_names status,bytes,vhost,path,rhost,agent,referer
 28 |       sql INSERT INTO accesslog (status,bytes,vhost,path,rhost,agent,referer) VALUES ($1,$2,$3,$4,$5,$6,$7)
 29 |       flush_intervals 5s
 30 |     </match>
 31 | 
 32 | 
 33 | 
 34 | ## Component
 35 | 
 36 | ### PostgresOutput
 37 | 
 38 | Plugin to store Postgres tables over SQL, to each columns per values, or to single column as json.
 39 | 
 40 | ## Configuration
 41 | 
 42 | ### MysqlOutput
 43 | 
 44 | MysqlOutput needs MySQL server's host/port/database/username/password, and INSERT format as SQL, or as table name and columns.
 45 | 
 46 |     <match output.by.sql.*>
 47 |       type mysql
 48 |       host master.db.service.local
 49 |       # port 3306 # default
 50 |       database application_logs
 51 |       username myuser
 52 |       password mypass
 53 |       key_names status,bytes,vhost,path,rhost,agent,referer
 54 |       sql INSERT INTO accesslog (status,bytes,vhost,path,rhost,agent,referer) VALUES (?,?,?,?,?,?,?)
 55 |       flush_intervals 5s
 56 |     </match>
 57 | 
 58 |     <match output.by.names.*>
 59 |       type mysql
 60 |       host master.db.service.local
 61 |       database application_logs
 62 |       username myuser
 63 |       password mypass
 64 |       key_names status,bytes,vhost,path,rhost,agent,referer
 65 |       table accesslog
 66 |       # 'columns' names order must be same with 'key_names'
 67 |       columns status,bytes,vhost,path,rhost,agent,referer
 68 |       flush_intervals 5s
 69 |     </match>
 70 | 
 71 | Or, insert json into single column.
 72 | 
 73 |     <match output.as.json.*>
 74 |       type mysql
 75 |       host master.db.service.local
 76 |       database application_logs
 77 |       username root
 78 |       table accesslog
 79 |       columns jsondata
 80 |       format json
 81 |       flush_intervals 5s
 82 |     </match>
 83 | 
 84 | To include time/tag into output, use `include_time_key` and `include_tag_key`, like this:
 85 | 
 86 |     <match output.with.tag.and.time.*>
 87 |       type mysql
 88 |       host my.mysql.local
 89 |       database anydatabase
 90 |       username yourusername
 91 |       password secret
 92 | 
 93 |       include_time_key yes
 94 |       ### default `time_format` is ISO-8601
 95 |       # time_format %Y%m%d-%H%M%S
 96 |       ### default `time_key` is 'time'
 97 |       # time_key timekey
 98 | 
 99 |       include_tag_key yes
100 |       ### default `tag_key` is 'tag'
101 |       # tag_key tagkey
102 | 
103 |       table anydata
104 |       key_names time,tag,field1,field2,field3,field4
105 |       sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
106 |     </match>
107 | 
108 | Or, for json:
109 | 
110 |     <match output.with.tag.and.time.as.json.*>
111 |       type mysql
112 |       host database.local
113 |       database foo
114 |       username root
115 | 
116 |       include_time_key yes
117 |       utc   # with UTC timezome output (default: localtime)
118 |       time_format %Y%m%d-%H%M%S
119 |       time_key timeattr
120 | 
121 |       include_tag_key yes
122 |       tag_key tagattr
123 |       table accesslog
124 |       columns jsondata
125 |       format json
126 |     </match>
127 |     #=> inserted json data into column 'jsondata' with addtional attribute 'timeattr' and 'tagattr'
128 | 
129 | ## TODO
130 | 
131 | * implement 'tag_mapped'
132 |   * dynamic tag based table selection
133 | 
134 | ## Copyright
135 | 
136 | * Copyright
137 |   * Copyright 2013 Uken Games
138 | * License
139 |   * Apache License, Version 2.0
140 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env rake
 2 | require "bundler/gem_tasks"
 3 | 
 4 | require 'rake/testtask'
 5 | Rake::TestTask.new(:test) do |test|
 6 |   test.libs << 'lib' << 'test'
 7 |   test.pattern = 'test/**/test_*.rb'
 8 |   test.verbose = true
 9 | end
10 | 
11 | task :default => :test
12 | 


--------------------------------------------------------------------------------
/fluent-plugin-postgres.gemspec:
--------------------------------------------------------------------------------
 1 | # -*- encoding: utf-8 -*-
 2 | Gem::Specification.new do |s|
 3 |   s.name          = 'fluent-plugin-postgres'
 4 |   s.version       = '0.1.0'
 5 |   s.authors       = ['TAGOMORI Satoshi', 'Diogo Terror', 'pitr']
 6 |   s.email         = ['team@uken.com']
 7 |   s.description   = %q{fluent plugin to insert on PostgreSQL}
 8 |   s.summary       = %q{fluent plugin to insert on PostgreSQL}
 9 |   s.homepage      = 'https://github.com/uken/fluent-plugin-postgres'
10 |   s.license       = 'Apache-2.0'
11 | 
12 |   s.files         = `git ls-files`.split($\)
13 |   s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
14 |   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
15 |   s.require_paths = ['lib']
16 | 
17 |   s.add_dependency 'fluentd', ['>= 0.14.15', '< 2']
18 |   s.add_dependency 'pg'
19 | 
20 |   s.add_development_dependency 'rake'
21 |   s.add_development_dependency 'test-unit', '~> 3.2.0'
22 | end
23 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/out_postgres.rb:
--------------------------------------------------------------------------------
 1 | require 'fluent/plugin/output'
 2 | require 'pg'
 3 | 
 4 | class Fluent::Plugin::PostgresOutput < Fluent::Plugin::Output
 5 |   Fluent::Plugin.register_output('postgres', self)
 6 | 
 7 |   helpers :inject, :compat_parameters
 8 | 
 9 |   config_param :host, :string
10 |   config_param :port, :integer, :default => nil
11 |   config_param :database, :string
12 |   config_param :username, :string
13 |   config_param :password, :string, :default => ''
14 | 
15 |   config_param :key_names, :string, :default => nil # nil allowed for json format
16 |   config_param :sql, :string, :default => nil
17 |   config_param :table, :string, :default => nil
18 |   config_param :columns, :string, :default => nil
19 | 
20 |   config_param :format, :string, :default => "raw" # or json
21 | 
22 |   attr_accessor :handler
23 | 
24 |   # We don't currently support mysql's analogous json format
25 |   def configure(conf)
26 |     compat_parameters_convert(conf, :inject)
27 |     super
28 | 
29 |     if @format == 'json'
30 |       @format_proc = Proc.new{|tag, time, record| record.to_json}
31 |     else
32 |       @key_names = @key_names.split(/\s*,\s*/)
33 |       @format_proc = Proc.new{|tag, time, record| @key_names.map{|k| record[k]}}
34 |     end
35 | 
36 |     if @columns.nil? and @sql.nil?
37 |       raise Fluent::ConfigError, "columns or sql MUST be specified, but missing"
38 |     end
39 |     if @columns and @sql
40 |       raise Fluent::ConfigError, "both of columns and sql are specified, but specify one of them"
41 |     end
42 |   end
43 | 
44 |   def start
45 |     super
46 |   end
47 | 
48 |   def shutdown
49 |     super
50 |   end
51 | 
52 |   def format(tag, time, record)
53 |     record = inject_values_to_record(tag, time, record)
54 |     [tag, time, @format_proc.call(tag, time, record)].to_msgpack
55 |   end
56 | 
57 |   def multi_workers_ready?
58 |     true
59 |   end
60 | 
61 |   def formatted_to_msgpack_binary?
62 |     true
63 |   end
64 | 
65 |   def client
66 |     PG::Connection.new({
67 |       :host => @host, :port => @port,
68 |       :user => @username, :password => @password,
69 |       :dbname => @database
70 |     })
71 |   end
72 | 
73 |   def write(chunk)
74 |     handler = self.client
75 |     handler.prepare("write", @sql)
76 |     chunk.msgpack_each { |tag, time, data|
77 |       handler.exec_prepared("write", data)
78 |     }
79 |     handler.close
80 |   end
81 | end
82 | 


--------------------------------------------------------------------------------
/test/helper.rb:
--------------------------------------------------------------------------------
 1 | require 'rubygems'
 2 | require 'bundler'
 3 | begin
 4 |   Bundler.setup(:default, :development)
 5 | rescue Bundler::BundlerError => e
 6 |   $stderr.puts e.message
 7 |   $stderr.puts "Run `bundle install` to install missing gems"
 8 |   exit e.status_code
 9 | end
10 | require 'test/unit'
11 | 
12 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13 | $LOAD_PATH.unshift(File.dirname(__FILE__))
14 | require 'fluent/test'
15 | require 'fluent/test/driver/output'
16 | require 'fluent/test/helpers'
17 | unless ENV.has_key?('VERBOSE')
18 |   nulllogger = Object.new
19 |   nulllogger.instance_eval {|obj|
20 |     def method_missing(method, *args)
21 |       # pass
22 |     end
23 |   }
24 |   $log = nulllogger
25 | end
26 | 
27 | require 'fluent/plugin/out_postgres'
28 | 
29 | class Test::Unit::TestCase
30 |   include Fluent::Test::Helpers
31 | end
32 | 


--------------------------------------------------------------------------------
/test/plugin/test_out_postgres.rb:
--------------------------------------------------------------------------------
  1 | require 'helper'
  2 | require 'pg'
  3 | 
  4 | class PostgresOutputTest < Test::Unit::TestCase
  5 |   def setup
  6 |     Fluent::Test.setup
  7 |   end
  8 | 
  9 |   CONFIG = %[
 10 | host database.local
 11 | database foo
 12 | username bar
 13 | password mogera
 14 | key_names field1,field2,field3
 15 | sql INSERT INTO baz (col1,col2,col3,col4) VALUES (?,?,?,?)
 16 |   ]
 17 | 
 18 |   def create_driver(conf=CONFIG)
 19 |     d = Fluent::Test::Driver::Output.new(Fluent::Plugin::PostgresOutput).configure(conf)
 20 |     d.instance.instance_eval {
 21 |       def client
 22 |         obj = Object.new
 23 |         obj.instance_eval {
 24 |           def prepare(*args); true; end
 25 |         def exec_prepared(*args); true; end
 26 |         def close; true; end
 27 |         }
 28 |         obj
 29 |       end
 30 |     }
 31 |     d
 32 |   end
 33 | 
 34 |   def test_configure_fails_if_both_cols_and_sql_specified
 35 |     assert_raise(Fluent::ConfigError) {
 36 |       create_driver %[
 37 | host database.local
 38 | database foo
 39 | username bar
 40 | password mogera
 41 | key_names field1,field2,field3
 42 | sql INSERT INTO baz (col1,col2,col3,col4) VALUES (?,?,?,?)
 43 | columns col1,col2,col3,col4
 44 |       ]
 45 |     }
 46 |   end
 47 | 
 48 |   def test_configure_fails_if_neither_cols_or_sql_specified
 49 |     assert_raise(Fluent::ConfigError) {
 50 |       create_driver %[
 51 | host database.local
 52 | database foo
 53 | username bar
 54 | password mogera
 55 | key_names field1,field2,field3
 56 |       ]
 57 |     }
 58 |   end
 59 | 
 60 |   def test_key_names_with_spaces
 61 |     d = create_driver %[
 62 | host database.local
 63 | database foo
 64 | username bar
 65 | password mogera
 66 | table baz
 67 | key_names time, tag, field1, field2, field3, field4
 68 | sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
 69 |     ]
 70 |     assert_equal ["time", "tag", "field1", "field2", "field3", "field4"], d.instance.key_names
 71 |   end
 72 | 
 73 |   def test_time_and_tag_key
 74 |     d = create_driver %[
 75 | host database.local
 76 | database foo
 77 | username bar
 78 | password mogera
 79 | include_time_key yes
 80 | utc
 81 | include_tag_key yes
 82 | table baz
 83 | key_names time,tag,field1,field2,field3,field4
 84 | sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
 85 |     ]
 86 |     assert_equal 'INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)', d.instance.sql
 87 | 
 88 |     time = event_time('2012-12-17 01:23:45 UTC')
 89 |     record = {'field1'=>'value1','field2'=>'value2','field3'=>'value3','field4'=>'value4'}
 90 |     d.run(default_tag: 'test') do
 91 |       d.feed(time, record)
 92 |     end
 93 |     assert_equal ['test', time, ['2012-12-17T01:23:45Z','test','value1','value2','value3','value4']].to_msgpack, d.formatted[0]
 94 |   end
 95 | 
 96 |   def test_time_and_tag_key_complex
 97 |     d = create_driver %[
 98 | host database.local
 99 | database foo
100 | username bar
101 | password mogera
102 | include_time_key yes
103 | utc
104 | time_format %Y%m%d-%H%M%S
105 | time_key timekey
106 | include_tag_key yes
107 | tag_key tagkey
108 | table baz
109 | key_names timekey,tagkey,field1,field2,field3,field4
110 | sql INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)
111 |     ]
112 |     assert_equal 'INSERT INTO baz (coltime,coltag,col1,col2,col3,col4) VALUES (?,?,?,?,?,?)', d.instance.sql
113 | 
114 |     time = event_time('2012-12-17 09:23:45 +0900')
115 |     record = {'field1'=>'value1','field2'=>'value2','field3'=>'value3','field4'=>'value4'}
116 |     d.run(default_tag: 'test') do
117 |       d.feed(time, record)
118 |     end
119 |     assert_equal ['test', time, ['20121217-002345','test','value1','value2','value3','value4']].to_msgpack, d.formatted[0]
120 |   end
121 | end
122 | 


--------------------------------------------------------------------------------