├── .gitignore
├── Gemfile
├── LICENSE.txt
├── README.md
├── Rakefile
├── fluent-plugin-dogstatsd.gemspec
├── lib
    └── fluent
    │   └── plugin
    │       ├── dogstatsd.rb
    │       ├── dogstatsd
    │           └── version.rb
    │       └── out_dogstatsd.rb
└── test
    ├── plugin
        └── test_out_dogstatsd.rb
    └── test_helper.rb


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.gem
 2 | *.rbc
 3 | .bundle
 4 | .config
 5 | .yardoc
 6 | Gemfile.lock
 7 | InstalledFiles
 8 | _yardoc
 9 | coverage
10 | doc/
11 | lib/bundler/man
12 | pkg
13 | rdoc
14 | spec/reports
15 | test/tmp
16 | test/version_tmp
17 | tmp
18 | *.bundle
19 | *.so
20 | *.o
21 | *.a
22 | mkmf.log
23 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | 
3 | # Specify your gem's dependencies in fluent-plugin-dogstatsd.gemspec
4 | gemspec
5 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2014 Ryota Arai
 2 | 
 3 | MIT License
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining
 6 | a copy of this software and associated documentation files (the
 7 | "Software"), to deal in the Software without restriction, including
 8 | without limitation the rights to use, copy, modify, merge, publish,
 9 | distribute, sublicense, and/or sell copies of the Software, and to
10 | permit persons to whom the Software is furnished to do so, subject to
11 | the following conditions:
12 | 
13 | The above copyright notice and this permission notice shall be
14 | included in all copies or substantial portions of the Software.
15 | 
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # fluent-plugin-dogstatsd
  2 | 
  3 | Fluend plugin for Dogstatsd, that is statsd server for Datadog.
  4 | 
  5 | ## Installation
  6 | 
  7 |     $ gem install fluent-plugin-dogstatsd
  8 | 
  9 | ## Usage
 10 | 
 11 | ```
 12 | $ echo '{"type": "increment", "key": "apache.requests", "tags": {"url": "/"}}' | fluent-cat dogstatsd.hello
 13 | $ echo '{"type": "histogram", "key": "apache.response_time", "value": 10.5, "tags": {"url": "/hello"}}' | fluent-cat dogstatsd.hello
 14 | $ echo '{"type": "event", "title": "Deploy", "text": "New revision"}' | fluent-cat dogstatsd.hello
 15 | ```
 16 | 
 17 | Supported types are `increment`, `decrement`, `count`, `gauge`, `histogram`, `timing`, `set` and `event`.
 18 | 
 19 | ## Configuration
 20 | 
 21 | ```
 22 | <match dogstatsd.*>
 23 |   type dogstatsd
 24 | 
 25 |   # Dogstatsd host
 26 |   host localhost
 27 | 
 28 |   # Dogstatsd port
 29 |   port 8125
 30 | 
 31 |   # Use tag of fluentd record as key sent to Dogstatsd
 32 |   use_tag_as_key false
 33 | 
 34 |   # (Treat fields in a record as tags)
 35 |   # flat_tags true
 36 | 
 37 |   # (Metric type in Datadog.)
 38 |   # metric_type increment
 39 | 
 40 |   # Default: "value"
 41 |   # value_key Value
 42 | </match>
 43 | ```
 44 | 
 45 | ## Example
 46 | 
 47 | ### Count log lines
 48 | 
 49 | ```apache
 50 | <source>
 51 |   type tail
 52 |   path /tmp/sample.log
 53 |   tag datadog.increment.sample
 54 |   format ...
 55 | </source>
 56 | 
 57 | <match datadog.increment.*>
 58 |   type dogstatsd
 59 |   metric_type increment
 60 |   flat_tags true
 61 |   use_tag_as_key true
 62 | </match>
 63 | ```
 64 | 
 65 | ### Histogram
 66 | 
 67 | ```apache
 68 | <source>
 69 |   type tail
 70 |   path /tmp/sample.log
 71 |   tag datadog.histogram.sample
 72 |   format /^(?<value>[^ ]*) (?<host>[^ ]*)$/
 73 | </source>
 74 | 
 75 | <match datadog.histogram.*>
 76 |   type dogstatsd
 77 |   metric_type histogram
 78 |   flat_tags true
 79 |   use_tag_as_key true
 80 | </match>
 81 | ```
 82 | 
 83 | ### MySQL threads
 84 | 
 85 | ```apache
 86 | <source>
 87 |   type mysql_query
 88 |   tag datadog.histogram.mysql_threads
 89 |   query SHOW VARIABLES LIKE 'Thread_%'
 90 | </source>
 91 | 
 92 | <match datadog.histogram.mysql_threads>
 93 |   type dogstatsd
 94 |   metric_type histogram
 95 |   value_key Value
 96 |   flat_tags true
 97 |   use_tag_as_key true
 98 | </match>
 99 | ```
100 | 
101 | ## Contributing
102 | 
103 | 1. Fork it ( https://github.com/ryotarai/fluent-plugin-dogstatsd/fork )
104 | 2. Create your feature branch (`git checkout -b my-new-feature`)
105 | 3. Commit your changes (`git commit -am 'Add some feature'`)
106 | 4. Push to the branch (`git push origin my-new-feature`)
107 | 5. Create a new Pull Request
108 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | require "bundler/gem_tasks"
 2 | 
 3 | require 'rake/testtask'
 4 | 
 5 | Rake::TestTask.new(:test) do |test|
 6 |   test.libs << 'test'
 7 |   test.test_files = FileList['test/plugin/*.rb']
 8 | end
 9 | 
10 | 


--------------------------------------------------------------------------------
/fluent-plugin-dogstatsd.gemspec:
--------------------------------------------------------------------------------
 1 | # coding: utf-8
 2 | lib = File.expand_path('../lib', __FILE__)
 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 4 | require 'fluent/plugin/dogstatsd/version'
 5 | 
 6 | Gem::Specification.new do |spec|
 7 |   spec.name          = "fluent-plugin-dogstatsd"
 8 |   spec.version       = Fluent::Plugin::Dogstatsd::VERSION
 9 |   spec.authors       = ["Ryota Arai"]
10 |   spec.email         = ["ryota.arai@gmail.com"]
11 |   spec.summary       = %q{Fluent plugin for Dogstatsd, that is statsd server for Datadog.}
12 |   spec.homepage      = "https://github.com/ryotarai/fluent-plugin-dogstatsd"
13 |   spec.license       = "MIT"
14 | 
15 |   spec.files         = `git ls-files -z`.split("\x0")
16 |   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17 |   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
18 |   spec.require_paths = ["lib"]
19 | 
20 |   spec.add_dependency "fluentd"
21 |   spec.add_dependency "dogstatsd-ruby", "~> 1.4.1"
22 | 
23 |   spec.add_development_dependency "bundler", "~> 1.6"
24 |   spec.add_development_dependency "rake"
25 |   spec.add_development_dependency "test-unit"
26 | end
27 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/dogstatsd.rb:
--------------------------------------------------------------------------------
 1 | require "fluent/plugin/dogstatsd/version"
 2 | 
 3 | module Fluent
 4 |   module Plugin
 5 |     module Dogstatsd
 6 |       # Your code goes here...
 7 |     end
 8 |   end
 9 | end
10 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/dogstatsd/version.rb:
--------------------------------------------------------------------------------
1 | module Fluent
2 |   module Plugin
3 |     module Dogstatsd
4 |       VERSION = "0.0.6"
5 |     end
6 |   end
7 | end
8 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/out_dogstatsd.rb:
--------------------------------------------------------------------------------
  1 | module Fluent
  2 |   class DogstatsdOutput < BufferedOutput
  3 |     Plugin.register_output('dogstatsd', self)
  4 | 
  5 |     config_param :host, :string, :default => nil
  6 |     config_param :port, :integer, :default => nil
  7 |     config_param :use_tag_as_key, :bool, :default => false
  8 |     config_param :use_tag_as_key_if_missing, :bool, :default => false
  9 |     config_param :flat_tags, :bool, :default => false
 10 |     config_param :flat_tag, :bool, :default => false # obsolete
 11 |     config_param :metric_type, :string, :default => nil
 12 |     config_param :value_key, :string, :default => nil
 13 |     config_param :sample_rate, :float, :default => nil
 14 | 
 15 |     unless method_defined?(:log)
 16 |       define_method(:log) { $log }
 17 |     end
 18 | 
 19 |     attr_accessor :statsd
 20 | 
 21 |     def initialize
 22 |       super
 23 | 
 24 |       require 'statsd' # dogstatsd-ruby
 25 |     end
 26 | 
 27 |     def start
 28 |       super
 29 | 
 30 |       host = @host || Statsd::DEFAULT_HOST
 31 |       port = @port || Statsd::DEFAULT_PORT
 32 | 
 33 |       @statsd ||= Statsd.new(host, port)
 34 |     end
 35 | 
 36 |     def format(tag, time, record)
 37 |       [tag, time, record].to_msgpack
 38 |     end
 39 | 
 40 |     def write(chunk)
 41 |       @statsd.batch do |s|
 42 |         chunk.msgpack_each do |tag, time, record|
 43 |           key = if @use_tag_as_key
 44 |                   tag
 45 |                 else
 46 |                   record.delete('key')
 47 |                 end
 48 | 
 49 |           if !key && @use_tag_as_key_if_missing
 50 |             key = tag
 51 |           end
 52 | 
 53 |           unless key
 54 |             log.warn "'key' is not specified. skip this record:", tag: tag
 55 |             next
 56 |           end
 57 | 
 58 |           value = record.delete(@value_key || 'value')
 59 | 
 60 |           options = {}
 61 |           title = record.delete('title')
 62 |           text  = record.delete('text')
 63 |           type  = @metric_type || record.delete('type')
 64 |           sample_rate = @sample_rate || record.delete('sample_rate')
 65 | 
 66 |           if sample_rate
 67 |             options[:sample_rate] = sample_rate
 68 |           end
 69 | 
 70 |           tags = if @flat_tags || @flat_tag
 71 |                    record
 72 |                  else
 73 |                    record['tags']
 74 |                  end
 75 |           if tags
 76 |             options[:tags] = tags.map do |k, v|
 77 |               "#{k}:#{v}"
 78 |             end
 79 |           end
 80 | 
 81 |           case type
 82 |           when 'increment'
 83 |             s.increment(key, options)
 84 |           when 'decrement'
 85 |             s.decrement(key, options)
 86 |           when 'count'
 87 |             s.count(key, value, options)
 88 |           when 'gauge'
 89 |             s.gauge(key, value, options)
 90 |           when 'histogram'
 91 |             s.histogram(key, value, options)
 92 |           when 'timing'
 93 |             s.timing(key, value, options)
 94 |           when 'set'
 95 |             s.set(key, value, options)
 96 |           when 'event'
 97 |             options[:alert_type] = record['alert_type']
 98 |             s.event(title, text, options)
 99 |           when nil
100 |             log.warn "type is not provided (You can provide type via `metric_type` in config or `type` field in a record."
101 |           else
102 |             log.warn "Type '#{type}' is unknown."
103 |           end
104 |         end
105 |       end
106 |     end
107 |   end
108 | end
109 | 


--------------------------------------------------------------------------------
/test/plugin/test_out_dogstatsd.rb:
--------------------------------------------------------------------------------
  1 | require 'test_helper'
  2 | 
  3 | class DummyStatsd
  4 |   attr_reader :messages
  5 | 
  6 |   def initialize
  7 |     @messages = []
  8 |   end
  9 | 
 10 |   def batch
 11 |     yield(self)
 12 |   end
 13 | 
 14 |   %i!increment decrement count gauge histogram timing set event!.each do |name|
 15 |     define_method(name) do |*args|
 16 |       @messages << [name, args].flatten
 17 |     end
 18 |   end
 19 | end
 20 | 
 21 | class DogstatsdOutputTest < Test::Unit::TestCase
 22 |   def setup
 23 |     Fluent::Test.setup
 24 |     require 'fluent/plugin/out_dogstatsd'
 25 |   end
 26 | 
 27 |   def teardown
 28 |   end
 29 | 
 30 |   def test_configure
 31 |     d = create_driver(<<-EOC)
 32 |       type dogstatsd
 33 |       host HOST
 34 |       port 12345
 35 |     EOC
 36 | 
 37 |     assert_equal('HOST', d.instance.host)
 38 |     assert_equal(12345, d.instance.port)
 39 |   end
 40 | 
 41 |   def test_write
 42 |     d = create_driver
 43 | 
 44 |     d.emit({'type' => 'increment', 'key' => 'hello.world1'}, Time.now.to_i)
 45 |     d.emit({'type' => 'increment', 'key' => 'hello.world2'}, Time.now.to_i)
 46 |     d.emit({'type' => 'decrement', 'key' => 'hello.world'}, Time.now.to_i)
 47 |     d.emit({'type' => 'count', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
 48 |     d.emit({'type' => 'gauge', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
 49 |     d.emit({'type' => 'histogram', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
 50 |     d.emit({'type' => 'timing', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
 51 |     d.emit({'type' => 'set', 'value' => 10, 'key' => 'hello.world'}, Time.now.to_i)
 52 |     d.emit({'type' => 'event', 'title' => 'Deploy', 'text' => 'Revision', 'key' => 'hello.world'}, Time.now.to_i)
 53 |     d.run
 54 | 
 55 |     assert_equal(d.instance.statsd.messages, [
 56 |       [:increment, 'hello.world1', {}],
 57 |       [:increment, 'hello.world2', {}],
 58 |       [:decrement, 'hello.world', {}],
 59 |       [:count, 'hello.world', 10, {}],
 60 |       [:gauge, 'hello.world', 10, {}],
 61 |       [:histogram, 'hello.world', 10, {}],
 62 |       [:timing, 'hello.world', 10, {}],
 63 |       [:set, 'hello.world', 10, {}],
 64 |       [:event, 'Deploy', 'Revision', {}],
 65 |     ])
 66 |   end
 67 | 
 68 |   def test_flat_tag
 69 |     d = create_driver(<<-EOC)
 70 | #{default_config}
 71 | flat_tag true
 72 |     EOC
 73 | 
 74 |     d.emit({'type' => 'increment', 'key' => 'hello.world', 'tagKey' => 'tagValue'}, Time.now.to_i)
 75 |     d.run
 76 | 
 77 |     assert_equal(d.instance.statsd.messages, [
 78 |       [:increment, 'hello.world', {tags: ["tagKey:tagValue"]}],
 79 |     ])
 80 |   end
 81 | 
 82 |   def test_metric_type
 83 |     d = create_driver(<<-EOC)
 84 | #{default_config}
 85 | metric_type decrement
 86 |     EOC
 87 | 
 88 |     d.emit({'key' => 'hello.world', 'tags' => {'tagKey' => 'tagValue'}}, Time.now.to_i)
 89 |     d.run
 90 | 
 91 |     assert_equal(d.instance.statsd.messages, [
 92 |       [:decrement, 'hello.world', {tags: ["tagKey:tagValue"]}],
 93 |     ])
 94 |   end
 95 | 
 96 |   def test_use_tag_as_key
 97 |     d = create_driver(<<-EOC)
 98 | #{default_config}
 99 | use_tag_as_key true
100 |     EOC
101 | 
102 |     d.emit({'type' => 'increment'}, Time.now.to_i)
103 |     d.run
104 | 
105 |     assert_equal(d.instance.statsd.messages, [
106 |       [:increment, 'dogstatsd.tag', {}],
107 |     ])
108 |   end
109 | 
110 |   def test_use_tag_as_key_fallback
111 |     d = create_driver(<<-EOC)
112 | #{default_config}
113 | use_tag_as_key_if_missing true
114 |     EOC
115 | 
116 |     d.emit({'type' => 'increment'}, Time.now.to_i)
117 |     d.run
118 | 
119 |     assert_equal(d.instance.statsd.messages, [
120 |       [:increment, 'dogstatsd.tag', {}],
121 |     ])
122 |   end
123 | 
124 |   def test_tags
125 |     d = create_driver
126 |     d.emit({'type' => 'increment', 'key' => 'hello.world', 'tags' => {'key' => 'value'}}, Time.now.to_i)
127 |     d.run
128 | 
129 |     assert_equal(d.instance.statsd.messages, [
130 |       [:increment, 'hello.world', {tags: ["key:value"]}],
131 |     ])
132 |   end
133 | 
134 |   def test_sample_rate_config
135 |     d = create_driver(<<-EOC)
136 | #{default_config}
137 | sample_rate .5
138 |     EOC
139 | 
140 |     d.emit({'type' => 'increment', 'key' => 'tag'}, Time.now.to_i)
141 |     d.run
142 | 
143 |     assert_equal(d.instance.statsd.messages, [
144 |       [:increment, 'tag', {sample_rate: 0.5}],
145 |     ])
146 |   end
147 | 
148 |   def test_sample_rate
149 |     d = create_driver
150 |     d.emit({'type' => 'increment', 'sample_rate' => 0.5, 'key' => 'tag'}, Time.now.to_i)
151 |     d.run
152 | 
153 |     assert_equal(d.instance.statsd.messages, [
154 |       [:increment, 'tag', {sample_rate: 0.5}],
155 |     ])
156 |   end
157 | 
158 |   private
159 |   def default_config
160 |     <<-EOC
161 |     type dogstatsd
162 |     EOC
163 |   end
164 | 
165 |   def create_driver(conf = default_config)
166 |     Fluent::Test::BufferedOutputTestDriver.new(Fluent::DogstatsdOutput, 'dogstatsd.tag').configure(conf).tap do |d|
167 |       d.instance.statsd = DummyStatsd.new
168 |     end
169 |   end
170 | end
171 | 
172 | 


--------------------------------------------------------------------------------
/test/test_helper.rb:
--------------------------------------------------------------------------------
1 | require 'test/unit'
2 | require 'fluent/test'
3 | 
4 | 


--------------------------------------------------------------------------------