├── Gemfile
├── .gitignore
├── Rakefile
├── test
    ├── helper.rb
    └── plugin
    │   ├── test_out_mqtt.rb
    │   └── test_in_mqtt.rb
├── README.md
├── fluent-plugin-mqtt.gemspec
├── LICENSE.txt
└── lib
    └── fluent
        └── plugin
            ├── in_mqtt.rb
            └── out_mqtt.rb


/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | 
3 | # Specify your gem's dependencies in fluent-plugin-mqtt.gemspec
4 | gemspec
5 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | *.gem
 2 | *.rbc
 3 | .bundle
 4 | .config
 5 | .yardoc
 6 | Gemfile.lock
 7 | InstalledFiles
 8 | _yardoc
 9 | coverage
10 | doc/
11 | lib/bundler/man
12 | pkg
13 | rdoc
14 | spec/reports
15 | test/tmp
16 | test/version_tmp
17 | tmp
18 | *~


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | require "bundler/gem_tasks"
 2 | require "rake/testtask"
 3 | 
 4 | Rake::TestTask.new(:test) do |test|
 5 |   test.libs << 'lib' << 'test'
 6 |   test.pattern = 'test/**/test_*.rb'
 7 |   test.verbose = true
 8 | end
 9 | 
10 | task :default => :test


--------------------------------------------------------------------------------
/test/helper.rb:
--------------------------------------------------------------------------------
 1 | require 'rubygems'
 2 | require 'bundler'
 3 | begin
 4 |   Bundler.setup(:default, :development)
 5 | rescue Bundler::BundlerError => e
 6 |   $stderr.puts e.message
 7 |   $stderr.puts "Run `bundle install` to install missing gems"
 8 |   exit e.status_code
 9 | end
10 | require 'test/unit'
11 | 
12 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13 | $LOAD_PATH.unshift(File.dirname(__FILE__))
14 | require 'fluent/test'
15 | require 'fluent/test/helpers'
16 | unless ENV.has_key?('VERBOSE')
17 |   nulllogger = Object.new
18 |   nulllogger.instance_eval {|obj|
19 |     def method_missing(method, *args)
20 |       # pass
21 |     end
22 |   }
23 |   $log = nulllogger
24 | end
25 | 
26 | require 'fluent/plugin/in_mqtt'
27 | require 'fluent/plugin/out_mqtt'
28 | 
29 | class Test::Unit::TestCase
30 | end
31 | 
32 | include Fluent::Test::Helpers
33 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Fluent::Plugin::Mqtt
 2 | 
 3 | Fluent plugin for MQTT protocol
 4 | 
 5 | ## Installation
 6 | 
 7 | Add this line to your application's Gemfile:
 8 | 
 9 |     gem 'fluent-plugin-mqtt'
10 | 
11 | And then execute:
12 | 
13 |     $ bundle
14 | 
15 | Or install it yourself as:
16 | 
17 |     $ gem install fluent-plugin-mqtt
18 | 
19 | ## Usage
20 | 
21 | This client works as ONLY MQTT client.
22 | MQTT topics array is set as "#".
23 | 
24 | ```
25 | 
26 | <source>
27 |   type mqtt
28 |   bind 127.0.0.1
29 |   port 1883
30 |   username username
31 |   password password
32 | </source>
33 | 
34 | ```
35 | 
36 | ### Changelog version 0.0.10
37 | 
38 |  - Support multiple subscriptions (put more coma separated topic in topics parameter)
39 | 
40 | ## Contributing
41 | 
42 | 1. Fork it ( http://github.com/yuuna/fluent-plugin-mqtt/fork )
43 | 2. Create your feature branch (`git checkout -b my-new-feature`)
44 | 3. Commit your changes (`git commit -am 'Add some feature'`)
45 | 4. Push to the branch (`git push origin my-new-feature`)
46 | 5. Create new Pull Request
47 | 


--------------------------------------------------------------------------------
/fluent-plugin-mqtt.gemspec:
--------------------------------------------------------------------------------
 1 | # coding: utf-8
 2 | lib = File.expand_path('../lib', __FILE__)
 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 4 | 
 5 | Gem::Specification.new do |spec|
 6 |   spec.name          = "fluent-plugin-mqtt"
 7 |   spec.version       = "0.0.10"
 8 |   spec.authors       = ["Yuuna Kurita"]
 9 |   spec.email         = ["yuuna.m@gmail.com"]
10 |   spec.summary       = %q{fluentd input plugin for mqtt server}
11 |   spec.description   = %q{fluentd input plugin for mqtt server}
12 |   spec.homepage      = "http://github.com/yuuna/fluent-plugin-mqtt"
13 |   spec.license       = "MIT"
14 | 
15 |   spec.files         = `git ls-files`.split($/)
16 |   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17 |   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
18 |   spec.require_paths = ["lib"]
19 | 
20 |   spec.add_runtime_dependency "mqtt", "~> 0.3.1"
21 |   spec.add_runtime_dependency "fluentd"
22 |   spec.add_runtime_dependency "yajl-ruby"
23 |   spec.add_runtime_dependency "test-unit"
24 |   spec.add_development_dependency "bundler", "~> 1.5"
25 |   spec.add_development_dependency "rake"
26 | end
27 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2013 Yuuna Kurita
 2 | 
 3 | MIT License
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining
 6 | a copy of this software and associated documentation files (the
 7 | "Software"), to deal in the Software without restriction, including
 8 | without limitation the rights to use, copy, modify, merge, publish,
 9 | distribute, sublicense, and/or sell copies of the Software, and to
10 | permit persons to whom the Software is furnished to do so, subject to
11 | the following conditions:
12 | 
13 | The above copyright notice and this permission notice shall be
14 | included in all copies or substantial portions of the Software.
15 | 
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/in_mqtt.rb:
--------------------------------------------------------------------------------
  1 | require 'mqtt'
  2 | require 'fluent/plugin/input'
  3 | require 'fluent/plugin/parser'
  4 | 
  5 | module Fluent::Plugin
  6 |   class MqttInput < Input
  7 |     Fluent::Plugin.register_input('mqtt', self)
  8 | 
  9 |     helpers :thread, :inject, :compat_parameters, :parser
 10 | 
 11 |     DEFAULT_PARSER_TYPE = 'none'
 12 | 
 13 |     config_set_default :include_tag_key, false
 14 |     config_set_default :include_time_key, true
 15 | 
 16 |     config_param :port, :integer, :default => 1883
 17 |     config_param :bind, :string, :default => '127.0.0.1'
 18 |     config_param :topics, :array, :default => ['#'], value_type: :string
 19 |     config_param :format, :string, :default => DEFAULT_PARSER_TYPE
 20 |     config_param :client_id, :string, :default => nil
 21 |     config_param :username, :string, :default => nil
 22 |     config_param :password, :string, :default => nil
 23 |     config_param :ssl, :bool, :default => nil
 24 |     config_param :ca, :string, :default => nil
 25 |     config_param :key, :string, :default => nil
 26 |     config_param :cert, :string, :default => nil
 27 | 
 28 |     config_section :parse do
 29 |       config_set_default :@type, DEFAULT_PARSER_TYPE
 30 |     end
 31 | 
 32 |     def configure(conf)
 33 |       compat_parameters_convert(conf, :inject, :parser)
 34 |       super
 35 |       configure_parser(conf)
 36 |     end
 37 | 
 38 |     def configure_parser(conf)
 39 |       @parser = parser_create(usage: 'in_mqtt_parser', type: @format, conf: conf)
 40 |     end
 41 | 
 42 |     # Return [time (if not available return now), message]
 43 |     def parse(message)
 44 |       @parser.parse(message) {|time, record|
 45 |         return (time || Fluent::Engine.now), record
 46 |       }
 47 |     end
 48 | 
 49 |     def start
 50 |       super
 51 |       log.debug "start mqtt #{@bind}"
 52 |       opts = {host: @bind,
 53 |               port: @port}
 54 |       opts[:client_id] = @client_id if @client_id
 55 |       opts[:username] =  @username if @username
 56 |       opts[:password] = @password if @password
 57 |       opts[:ssl] = @ssl if @ssl
 58 |       opts[:ca_file] = @ca if @ca
 59 |       opts[:cert_file] = @cert if @cert
 60 |       opts[:key_file] = @key if @key
 61 |       @connect = MQTT::Client.connect(opts)
 62 |       log.debug "subscribing on topics #{@topics}"
 63 |       @topics.each do |topic|
 64 |         @connect.subscribe(topic)
 65 |       end
 66 | 
 67 |       thread_create(:in_mqtt_worker) do
 68 |         @connect.get do |topic,message|
 69 |           topic.gsub!("/","\.")
 70 |           log.debug "#{topic}: #{message}"
 71 |           begin
 72 |             time, record = self.parse(message)
 73 |             record = inject_values_to_record(topic, time, record)
 74 |           rescue Exception => e
 75 |             log.error e
 76 |           end
 77 |           emit topic, record, time
 78 |         end
 79 |       end
 80 |     end
 81 | 
 82 | 
 83 |     def emit topic, message, time = Fluent::Engine.now
 84 |       if message.class == Array
 85 |         message.each do |data|
 86 |           log.debug "#{topic}: #{data}"
 87 |           router.emit(topic , time , data)
 88 |         end
 89 |       else
 90 |         router.emit(topic , time , message)
 91 |       end
 92 |     end
 93 | 
 94 |     def shutdown
 95 |       @connect.disconnect
 96 |       super
 97 |     end
 98 |   end
 99 | end
100 | 


--------------------------------------------------------------------------------
/test/plugin/test_out_mqtt.rb:
--------------------------------------------------------------------------------
  1 | require_relative '../helper'
  2 | require 'fluent/test/driver/output'
  3 | 
  4 | class MqttOutputTest < Test::Unit::TestCase
  5 |   def setup
  6 |     Fluent::Test.setup
  7 |   end
  8 | 
  9 |   CONFIG = %[ bind 127.0.0.1
 10 |               port 1883
 11 |               format json ]
 12 | 
 13 |   def create_driver(conf = CONFIG)
 14 |     Fluent::Test::Driver::Output.new(Fluent::Plugin::OutMqtt).configure(conf)
 15 |   end
 16 | 
 17 |   def sub_client(topic = "td-agent/#")
 18 |     connect = MQTT::Client.connect("localhost")
 19 |     connect.subscribe(topic)
 20 |     return connect
 21 |   end
 22 | 
 23 |   def test_configure_1
 24 |     d = create_driver(
 25 |       %[ bind 127.0.0.1
 26 |          port 1883
 27 |          format json]
 28 |     )
 29 |     assert_equal '127.0.0.1', d.instance.bind
 30 |     assert_equal 1883, d.instance.port
 31 |     assert_equal 'json', d.instance.instance_variable_get(:@format)
 32 | 
 33 |     d2 = create_driver(
 34 |       %[ bind 127.0.0.1
 35 |          port 1883
 36 |          format csv
 37 |          fields time,message
 38 |          time_key time]
 39 |     )
 40 |     assert_equal 'csv', d2.instance.instance_variable_get(:@format)
 41 |     assert_equal 'time', d2.instance.inject_config.time_key
 42 |   end
 43 | 
 44 |   class TestWithTimeZone < self
 45 |     def setup
 46 |       @timeZone = ENV['TZ']
 47 |     end
 48 |     def teardown
 49 |       ENV['TZ'] = @timeZone
 50 |     end
 51 | 
 52 |     def test_format_csv
 53 |       ENV['TZ'] = 'Asia/Tokyo'
 54 | 
 55 |       d = create_driver(
 56 |         %[ bind 127.0.0.1
 57 |            port 1883
 58 |            format csv
 59 |            time_type string
 60 |            time_format %Y-%m-%dT%H:%M:%S%z
 61 |            fields time,message]
 62 |       )
 63 | 
 64 |       client = sub_client
 65 |       time = event_time("2011-01-02 13:14:15 UTC")
 66 |       data = [
 67 |         {tag: "tag1", message: "#{time},hello world" },
 68 |         {tag: "tag2", message: "#{time},hello to you to" },
 69 |         {tag: "tag3", message: "#{time}," },
 70 |       ]
 71 | 
 72 |       d.run(default_tag: "test") do
 73 |         data.each do |record|
 74 |           d.feed(time, record)
 75 |         end
 76 |       end
 77 |       3.times do |i|
 78 |         record = client.get
 79 |         assert_equal "td-agent", record[0]
 80 |         assert_equal "\"2011-01-02T22:14:15+0900\",\"#{data[i][:message]}\"\n", record[1]
 81 |       end
 82 |     end
 83 | 
 84 |     def test_format_json
 85 |       ENV['TZ'] = 'Asia/Tokyo'
 86 | 
 87 |       d = create_driver(
 88 |         %[ bind 127.0.0.1
 89 |            port 1883
 90 |            format json
 91 |            time_type string
 92 |            time_format %Y-%m-%dT%H:%M:%S%z
 93 |            fields time,message]
 94 |       )
 95 | 
 96 |       client = sub_client
 97 |       time = event_time("2011-01-02 13:14:15 UTC")
 98 |       data = [
 99 |         {tag: "tag1", message: "#{time},hello world" },
100 |         {tag: "tag2", message: "#{time},hello to you to" },
101 |         {tag: "tag3", message: "#{time}," },
102 |       ]
103 | 
104 |       d.run(default_tag: "test") do
105 |         data.each do |record|
106 |           d.feed(time, record)
107 |         end
108 |       end
109 |       3.times do |i|
110 |         record = client.get
111 |         assert_equal "td-agent", record[0]
112 |         assert_equal "{\"tag\":\"#{data[i][:tag]}\",\
113 | \"message\":\"#{data[i][:message]}\",\
114 | \"time\":\"2011-01-02T22:14:15+0900\"}\n", record[1]
115 |       end
116 |     end
117 |   end
118 | end
119 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/out_mqtt.rb:
--------------------------------------------------------------------------------
  1 | require 'mqtt'
  2 | require 'msgpack'
  3 | require 'fluent/plugin/output'
  4 | 
  5 | module Fluent::Plugin
  6 |   class OutMqtt < Output
  7 |     Fluent::Plugin.register_output('mqtt', self)
  8 | 
  9 |     helpers :compat_parameters, :inject, :formatter
 10 | 
 11 |     DEFAULT_BUFFER_TYPE = "memory"
 12 | 
 13 |     config_set_default :include_tag_key, false
 14 |     config_set_default :include_time_key, true
 15 | 
 16 |     config_param :port, :integer, :default => 1883
 17 |     config_param :bind, :string, :default => '127.0.0.1'
 18 |     config_param :topic, :string, :default => 'td-agent'
 19 |     config_param :format, :string, :default => 'none'
 20 |     config_param :client_id, :string, :default => nil
 21 |     config_param :username, :string, :default => nil
 22 |     config_param :password, :string, :default => nil
 23 |     config_param :ssl, :bool, :default => nil
 24 |     config_param :ca, :string, :default => nil
 25 |     config_param :key, :string, :default => nil
 26 |     config_param :cert, :string, :default => nil
 27 |     config_param :retain, :string, :default => true
 28 | 
 29 |     config_section :buffer do
 30 |       config_set_default :@type, DEFAULT_BUFFER_TYPE
 31 |       config_set_default :chunk_keys, ['tag']
 32 |     end
 33 | 
 34 |     config_section :inject do
 35 |       config_set_default :time_key, "time"
 36 |       config_set_default :time_type, "string"
 37 |       config_set_default :time_format, "%Y-%m-%dT%H:%M:%S%z"
 38 |     end
 39 | 
 40 |     def initialize
 41 |       super
 42 | 
 43 |       @clients = {}
 44 |       @connection_options = {}
 45 |       @collection_options = {:capped => false}
 46 |     end
 47 | 
 48 |     def configure(conf)
 49 |       compat_parameters_convert(conf, :buffer, :inject, :formatter)
 50 |       super
 51 |       @bind ||= conf['bind']
 52 |       @topic ||= conf['topic']
 53 |       @port ||= conf['port']
 54 |       @formatter = formatter_create
 55 |       if conf.has_key?('buffer_chunk_limit')
 56 |         #check buffer_size
 57 |         conf['buffer_chunk_limit'] = available_buffer_chunk_limit(conf)
 58 |       end
 59 |     end
 60 | 
 61 |     def start
 62 | 
 63 |       log.debug "start mqtt #{@bind}"
 64 |       opts = {host: @bind,
 65 |               port: @port}
 66 |       opts[:client_id] = @client_id if @client_id
 67 |       opts[:username] =  @username if @username
 68 |       opts[:password] = @password if @password
 69 |       opts[:ssl] = @ssl if @ssl
 70 |       opts[:ca_file] = @ca if @ca
 71 |       opts[:cert_file] = @cert if @cert
 72 |       opts[:key_file] = @key if @key
 73 |       @connect = MQTT::Client.connect(opts)
 74 |       super
 75 |     end
 76 | 
 77 |     def shutdown
 78 |       @connect.disconnect
 79 |       super
 80 |     end
 81 | 
 82 |     def format(tag, time, record)
 83 |       [time, record].to_msgpack
 84 |     end
 85 | 
 86 |     def formatted_to_msgpack_binary
 87 |       true
 88 |     end
 89 | 
 90 |     def multi_workers_ready?
 91 |       true
 92 |     end
 93 | 
 94 |     def write(chunk)
 95 |       tag = chunk.metadata.tag
 96 |       chunk.msgpack_each { |time, record|
 97 |         record = inject_values_to_record(tag, time, record)
 98 |         log.debug "write #{@topic} #{@formatter.format(tag,time,record)}"
 99 |         @connect.publish(@topic, @formatter.format(tag,time,record), retain=@retain)
100 |       }
101 |     end
102 | 
103 |     private
104 |     # Following limits are heuristic. BSON is sometimes bigger than MessagePack and JSON.
105 |     LIMIT_MQTT = 2 * 1024  # 2048kb
106 | 
107 |     def available_buffer_chunk_limit(conf)
108 |       if conf['buffer_chunk_limit'] > LIMIT_MQTT
109 |         log.warn ":buffer_chunk_limit(#{conf['buffer_chunk_limit']}) is large. Reset :buffer_chunk_limit with #{LIMIT_MQTT}"
110 |         LIMIT_MQTT
111 |       else
112 |         conf['buffer_chunk_limit']
113 |       end
114 |     end
115 |   end
116 | end
117 | 


--------------------------------------------------------------------------------
/test/plugin/test_in_mqtt.rb:
--------------------------------------------------------------------------------
  1 | require 'helper'
  2 | require 'fluent/test/driver/input'
  3 | 
  4 | class Fluent::Plugin::MqttInput
  5 |   #def emit topic, message , time = Fluent::Engine.now
  6 |     #if message.class == Array
  7 |       #message.each do |data|
  8 |         #$log.debug "#{topic}: #{data}"
  9 |         #Fluent::Engine.emit(topic, message["t"], data)
 10 |       #end
 11 |     #else
 12 |       #Fluent::Engine.emit(topic, message["t"], message)
 13 |     #end
 14 |   #end
 15 | 
 16 | end
 17 | 
 18 | include Fluent::Test::Helpers
 19 | 
 20 | class MqttInputTest < Test::Unit::TestCase
 21 |   def setup
 22 |     Fluent::Test.setup
 23 |   end
 24 | 
 25 |  CONFIG = %[ bind 127.0.0.1
 26 |              port 1883
 27 |              format json ]
 28 | 
 29 |   def create_driver(conf = CONFIG)
 30 |     Fluent::Test::Driver::Input.new(Fluent::Plugin::MqttInput).configure(conf)
 31 |   end
 32 | 
 33 |   def test_configure_1
 34 |     d = create_driver(
 35 |       %[ bind 127.0.0.1
 36 |          port 1883
 37 |          format none]
 38 |     )
 39 |     assert_equal '127.0.0.1', d.instance.bind
 40 |     assert_equal 1883, d.instance.port
 41 |     assert_equal 'none', d.instance.format
 42 | 
 43 |     d2 = create_driver(
 44 |       %[ bind 127.0.0.1
 45 |          port 1883
 46 |          format csv
 47 |          keys time,message
 48 |          time_key time]
 49 |     )
 50 |     assert_equal 'csv', d2.instance.format
 51 |     assert_equal 'time', d2.instance.inject_config.time_key
 52 |   end
 53 | 
 54 |   def test_configure_2
 55 |     d = create_driver(
 56 |       %[ bind 127.0.0.1
 57 |          port 1883
 58 |          format csv
 59 |          keys time,message ]
 60 |     )
 61 |     assert_equal '127.0.0.1', d.instance.bind
 62 |     assert_equal 1883, d.instance.port
 63 |     assert_equal 'csv', d.instance.format
 64 |   end
 65 | 
 66 |   def sub_client
 67 |     connect = MQTT::Client.connect("localhost")
 68 |     connect.subscribe('#')
 69 |     return connect
 70 |   end
 71 | 
 72 |   def test_format_json_without_time_key
 73 |     d = create_driver(
 74 |       %[ bind 127.0.0.1
 75 |          port 1883
 76 |          format json ]
 77 |     )
 78 |     time = event_time("2011-01-02 13:14:15 UTC")
 79 |     data = [
 80 |       {tag: "tag1", message: {"t" => time, "v" => {"a"=>1}}},
 81 |       {tag: "tag2", message: {"t" => time, "v" => {"a"=>1}}},
 82 |       {tag: "tag3", message: {"t" => time, "v" => {"a"=>32}}},
 83 |     ]
 84 | 
 85 |     d.run(expect_emits: 3, timeout: 5) do
 86 |       data.each do |record|
 87 |         send_data record[:tag], record[:message], d.instance.format
 88 |       end
 89 |     end
 90 | 
 91 |     emits = d.events
 92 |     assert_equal('tag1', emits[0][0])
 93 |     assert_equal({"t" => time, "v" => {"a"=>1}}, emits[0][2])
 94 | 
 95 |     assert_equal('tag2', emits[1][0])
 96 |     assert_equal({"t" => time, "v" => {"a"=>1}}, emits[1][2])
 97 | 
 98 |     assert_equal('tag3', emits[2][0])
 99 |     assert_equal({"t" => time, "v" => {"a"=>32}}, emits[2][2])
100 |   end
101 | 
102 |   def test_format_json_with_time_key
103 |     d = create_driver(
104 |       %[ bind 127.0.0.1
105 |          port 1883
106 |          format json
107 |          time_key t ]
108 |     )
109 |     time = event_time("2011-01-02 13:14:15 UTC")
110 |     data = [
111 |       {tag: "tag1", message: {"t" => time, "v" => {"a"=>1}}},
112 |       {tag: "tag2", message: {"t" => time, "v" => {"a"=>1}}},
113 |       {tag: "tag3", message: {"t" => time, "v" => {"a"=>31}}},
114 |       {tag: "tag3", message: {"t" => time, "v" => {"a"=>32}}},
115 |     ]
116 | 
117 |     d.run(expect_emits: 4, timeout: 5) do
118 |       data.each do |record|
119 |         send_data record[:tag], record[:message], d.instance.format
120 |       end
121 |     end
122 | 
123 |     emits = d.events
124 |     assert_equal('tag1', emits[0][0])
125 |     assert_equal(time, emits[0][1])
126 |     assert_equal({"v" => {"a"=>1}, "t" => time}, emits[0][2])
127 |   end
128 | 
129 |   def test_format_none
130 |     d = create_driver(
131 |       %[ bind 127.0.0.1
132 |          port 1883
133 |          format none]
134 |     )
135 | 
136 |     data = [
137 |       {tag: "tag1", message: 'hello world'},
138 |       {tag: "tag2", message: 'another world'},
139 |       {tag: "tag3", message: ''},
140 |     ]
141 | 
142 |     d.run(expect_emits: 3, timeout: 5) do
143 |       data.each do |record|
144 |         send_data record[:tag], record[:message], d.instance.format
145 |       end
146 |     end
147 | 
148 |     emits = d.events
149 |     time = Fluent::Engine.now
150 |     assert_equal('tag1', emits[0][0])
151 |     assert_equal({'message' => 'hello world'}, emits[0][2])
152 |     assert_equal('tag2', emits[1][0])
153 |     assert_equal({'message' => 'another world'}, emits[1][2])
154 |     assert_equal('tag3', emits[2][0])
155 |     assert_equal({'message' => ''}, emits[2][2])
156 |   end
157 | 
158 |   def test_format_csv
159 |     d = create_driver(
160 |       %[ bind 127.0.0.1
161 |          port 1883
162 |          format csv
163 |          keys time,message]
164 |     )
165 | 
166 |     time = event_time("2011-01-02 13:14:15 UTC")
167 |     data = [
168 |       {tag: "tag1", message: "#{time},hello world" },
169 |       {tag: "tag2", message: "#{time},hello to you to" },
170 |       {tag: "tag3", message: "#{time}," },
171 |     ]
172 | 
173 |     d.run(expect_emits: 3, timeout: 5) do
174 |       data.each do |record|
175 |         send_data record[:tag], record[:message], d.instance.format
176 |       end
177 |     end
178 | 
179 |     emits = d.events
180 |     #puts 'emits length', emits.length.to_s
181 |     assert_equal('tag1', emits[0][0])
182 |     assert_equal({'time' => time.to_s, 'message' => 'hello world'}, emits[0][2])
183 | 
184 |     assert_equal('tag2', emits[1][0])
185 |     assert_equal({'time' => time.to_s, 'message' => 'hello to you to'}, emits[1][2])
186 |     assert_equal('tag3', emits[2][0])
187 |     assert_equal({'time' => time.to_s, 'message' => nil}, emits[2][2])
188 |   end
189 | 
190 |   def test_format_csv_with_time_key
191 |     d = create_driver(
192 |       %[ bind 127.0.0.1
193 |          port 1883
194 |          format csv
195 |          keys time2,message
196 |          time_key time2
197 |          time_format %s]
198 |     )
199 | 
200 |     time = event_time("2011-01-02 13:14:15 UTC").to_i # to obtain unixtime stamp
201 |     data = [
202 |       {tag: "tag1", message: "#{time},abc" },
203 |       {tag: "tag2", message: "#{time},def" },
204 |       {tag: "tag3", message: "#{time},ghi" },
205 |       {tag: "tag3", message: "#{time}," },
206 |     ]
207 | 
208 |     d.run(expect_emits: 4, timeout: 5) do
209 |       data.each do |record|
210 |         send_data record[:tag], record[:message], d.instance.format
211 |       end
212 |     end
213 | 
214 |     emits = d.events
215 |     assert_equal('tag1', emits[0][0])
216 |     assert_equal({'message' => 'abc', "time2" => time}, emits[0][2])
217 | 
218 |     assert_equal('tag2', emits[1][0])
219 |     assert_equal({'message' => 'def', "time2" => time}, emits[1][2])
220 | 
221 |     assert_equal('tag3', emits[2][0])
222 |     assert_equal({'message' => 'ghi', "time2" => time}, emits[2][2])
223 | 
224 |     assert_equal('tag3', emits[3][0])
225 |     assert_equal({'message' => nil, "time2"=> time}, emits[3][2])
226 |   end
227 | 
228 |   def send_data tag, record, format
229 |     case format
230 |       when 'none'
231 |         sub_client.publish(tag, record)
232 |       when 'json'
233 |         sub_client.publish(tag, record.to_json)
234 |       when 'csv'
235 |         sub_client.publish(tag, record)
236 |       else
237 |         sub_client.publish(tag, record)
238 |     end
239 |   end
240 | end
241 | 


--------------------------------------------------------------------------------