├── .gitignore ├── .rspec ├── .rubocop.yml ├── .travis.yml ├── CONTRIBUTING.md ├── Gemfile ├── Guardfile ├── MIT-LICENSE.txt ├── README.md ├── Rakefile ├── config_relay.json ├── lib ├── sensu │ └── extensions │ │ ├── handlers │ │ └── relay.rb │ │ └── mutators │ │ └── metrics.rb └── version.rb ├── spec ├── conf.d │ ├── client.json │ ├── config_redis.json │ ├── config_relay.json │ ├── handler_metric_store.json │ └── rabbitmq.json ├── fixtures.rb ├── fixtures │ ├── events │ │ ├── bad_graphite.json │ │ ├── empty.json │ │ ├── event.json │ │ ├── graphite.json │ │ ├── json.json │ │ ├── json_withname.json │ │ ├── mutated.json │ │ └── opentsdb.json │ ├── settings.json │ └── settings_bad.json ├── helpers.rb ├── lib │ └── sensu │ │ └── extensions │ │ ├── handlers │ │ └── relay_spec.rb │ │ └── mutators │ │ └── metrics_spec.rb └── spec_helper.rb └── wizardvan.gemspec /.gitignore: -------------------------------------------------------------------------------- 1 | coverage 2 | Gemfile.lock 3 | .coveralls.yml 4 | tmp/ 5 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper 2 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | LineLength: 2 | Enabled: true 3 | Max: 128 4 | 5 | Encoding: 6 | Enabled: false 7 | 8 | CaseIndentation: 9 | Enabled: false 10 | 11 | MethodLength: 12 | Enabled: false 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 1.9.3 4 | - 2.0.0 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | 1. Visit #sensu on Freenode to discuss 4 | 2. Fork https://github.com/opower/sensu-metrics-relay 5 | 3. Create your feature branch (`git checkout -b my-new-feature`) 6 | 4. Commit your changes (`git commit -am 'Added some feature with tests'`) 7 | 5. Push your feature branch (`git push origin my-new-feature`) 8 | 6. Create a Pull Request 9 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | # vim: ft=ruby 2 | # More info at https://github.com/guard/guard#readme 3 | # 4 | # More info also at https://github.com/guard/guard-rspec -- this one in 5 | # particular details configuration options such as whether to run all tests 6 | # after a failing test starts passing 7 | 8 | guard :rspec do 9 | watch(%r{^spec/.+_spec\.rb}) 10 | watch(%r{^lib/(.+)\.rb$}) do |m| 11 | "spec/lib/#{m[1]}_spec.rb" 12 | end 13 | watch('spec/spec_helper.rb') { "spec" } 14 | watch('spec/fixtures.rb') { "spec" } 15 | watch(%r{^spec/fixtures/}) { "spec" } 16 | end 17 | -------------------------------------------------------------------------------- /MIT-LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Opower, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # WizardVan: A Sensu Metrics Relay 2 | 3 | [![Coverage Status](https://coveralls.io/repos/opower/sensu-metrics-relay/badge.png?branch=master)](https://coveralls.io/r/opower/sensu-metrics-relay?branch=master) [![Build Status](https://travis-ci.org/grepory/wizardvan.svg)](https://travis-ci.org/grepory/wizardvan) 4 | 5 | 6 | WizardVan uses a combination of a handler and a metric extension to open 7 | persistent TCP connections between the Sensu servers and back-end metric store 8 | for firehose relaying of metrics events from Sensu to metric stores. 9 | 10 | Handler extensions are useful in firehose situations. You could easily write a 11 | handler plug-in to do this, but an extension would be significantly less 12 | resource intensive. Were a simple firehose implemented in a plug-in, every 13 | event would cause a fork of the parent ruby process and writing data over a TCP 14 | (more likely) or UDP connection. 15 | 16 | An extension, on the other hand, could maintain persistent TCP connections to 17 | the firehose destination and simply write serialized event data over that 18 | connection. In high-volume metrics installations, 10-100k metrics per minute or 19 | more, this kind of optimization is a requirement. 20 | 21 | It also includes some mutation functionality to mutate from Graphite 22 | to OpenTSDB as well as accepts a new metric format for submitting metrics 23 | in JSON and mutating from that to any number of configured metric 24 | backends. To implement a mutator you simply need to define the endpoint 25 | in the relay configuration and then define a mutator in metrics.rb. 26 | 27 | **NOTE**: If you are pushing metrics in a format other than Graphite, you will 28 | have to to add "output\_type": "opentsdb" or "json" to your check definitions. 29 | 30 | ## Installation 31 | 32 | As root: 33 | 34 | 1. cp -R lib/sensu/extensions/\* /etc/sensu/extensions 35 | 2. cp config\_relay.json /etc/sensu/conf.d 36 | 3. Edit the values in config\_relay.json according to your environment. 37 | 4. Restart sensu-server 38 | 39 | Relay currently supports "graphite" and "opentsdb" as metric back-ends. 40 | 41 | ## Configuration 42 | 43 | config\_relay.json: 44 | 45 | ```json 46 | { 47 | "relay": { 48 | "graphite": { 49 | "host": "127.0.0.1", 50 | "port": 60000, 51 | "max_queue_size": 16384 # optional 52 | }, 53 | "opentsdb": { 54 | "host": "127.0.0.1", 55 | "port": 4424, 56 | "max_queue_size": 0, 57 | } 58 | } 59 | } 60 | ``` 61 | 62 | The metrics relay uses the plaintext protocol for Graphite and OpenTSDB's RPC protocol 63 | for transmitting metrics. Be sure to configure accordingly. 64 | 65 | The behavior of the mutator and handler is heavily influenced by several configuration 66 | options. The mutator introduces the concept of "output_type". By default, the extension 67 | assumes an "output_type" of "graphite." If your metrics are outputting anything besides 68 | graphite-formatted metrics, you will need to specify an "output_type" in your metrics 69 | checks that corresponds to a configured relay endpoint. 70 | 71 | In the example check below, you will also see the "auto_tag_host" configuration variable. 72 | Most metrics will be associated with a particular host, so this will default to yes. 73 | 74 | You must also specify "relay" as your handler. 75 | 76 | For example, if you have a metric check that outputs opentsdb: 77 | 78 | ```json 79 | "checks": { 80 | "my_check": { 81 | "name": "opentsdb_metric", 82 | "output_type": "opentsdb", 83 | "auto_tag_host": "yes", 84 | "type": "metric", 85 | "handlers": [ "relay" ], 86 | "command": "/path/to/opentsdb_metric.rb" 87 | } 88 | } 89 | ``` 90 | 91 | You would then configure "opentsdb" as an endpoint in relay. 92 | 93 | In the configuration example above, you will notice that we define graphite and opentsdb 94 | back-ends. The relay currently includes functionality to mutate between graphite and opentsdb. 95 | So, if you are submitting graphite metrics and define an opentsdb backend in config_relay.json, 96 | then it will automatically mutate from graphite to opentsdb and submit metrics to opentsdb. 97 | 98 | An optional `max_queue_size` parameter may be passed to an endpoint that will control the size 99 | of the buffer used to hold metrics until they are flushed to the network. It's recommended not 100 | to increase this beyond 16384--as this is the amount of data that EventMachine will write to the 101 | network in a single tick. 102 | 103 | ## JSON Metric format 104 | 105 | The relay currently only supports graphite and opentsdb as endpoints for 106 | mutation from JSON. The JSON format for metrics is as follows: 107 | 108 | ```json 109 | { 110 | "name": "metric_name", 111 | "value": 1, 112 | # optional epoch time stamp, precision can be higher, but graphite will floor() 113 | # and does not aggregate metrics, currently. 114 | "timestamp": 1365624303 115 | "tags": { 116 | "service": "some_service", # optional 117 | "some_tag": "some_value" # optional 118 | } 119 | } 120 | ``` 121 | 122 | ## Development 123 | 124 | ### Running the test suite 125 | 126 | To run tests continuously with guard, do: 127 | 128 | ``` 129 | bundle exec guard 130 | ``` 131 | 132 | Guard will start up a foreground process that runs the relevant specs every 133 | time you save a file and runs all the (non-slow) specs if you hit enter. For 134 | fancier interactions and configuration information, see the [guard 135 | documentation](https://github.com/guard/guard) and the [guard-rspec 136 | documentation](https://github.com/guard/guard-rspec). 137 | 138 | To run the full suite (this takes up to 10 minutes), do: 139 | 140 | ``` 141 | bundle exec rspec 142 | ``` 143 | 144 | For information on contributing, please see CONTRIBUTING.md 145 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | require 'rspec/core/rake_task' 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task :default => :spec 7 | -------------------------------------------------------------------------------- /config_relay.json: -------------------------------------------------------------------------------- 1 | { 2 | "relay": { 3 | "graphite": { 4 | "host": "127.0.0.1", 5 | "port": 60000 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /lib/sensu/extensions/handlers/relay.rb: -------------------------------------------------------------------------------- 1 | # ExponentialDecayTimer 2 | # 3 | # Implement an exponential backoff timer for reconnecting to metrics 4 | # backends. 5 | class ExponentialDecayTimer 6 | attr_accessor :reconnect_time 7 | 8 | def initialize 9 | @reconnect_time = 0 10 | end 11 | 12 | def get_reconnect_time(max_reconnect_time, connection_attempt_count) 13 | if @reconnect_time < max_reconnect_time 14 | seconds = @reconnect_time + (2**(connection_attempt_count - 1)) 15 | seconds = seconds * (0.5 * (1.0 + rand)) 16 | @reconnect_time = if seconds <= max_reconnect_time 17 | seconds 18 | else 19 | max_reconnect_time 20 | end 21 | end 22 | @reconnect_time 23 | end 24 | end 25 | 26 | module Sensu::Extension 27 | # Setup some basic error handling and connection management. Climb on top 28 | # of Sensu logging capability to log error states. 29 | class RelayConnectionHandler < EM::Connection 30 | 31 | # XXX: These should be runtime configurable. 32 | MAX_RECONNECT_ATTEMPTS = 10 33 | MAX_RECONNECT_TIME = 300 # seconds 34 | 35 | attr_accessor :message_queue, :connection_pool 36 | attr_accessor :name, :host, :port, :connected 37 | attr_accessor :reconnect_timer 38 | 39 | # ignore :reek:TooManyStatements 40 | def post_init 41 | @is_closed = false 42 | @connection_attempt_count = 0 43 | @max_reconnect_time = MAX_RECONNECT_TIME 44 | @comm_inactivity_timeout = 0 # disable inactivity timeout 45 | @pending_connect_timeout = 30 # seconds 46 | @reconnect_timer = ExponentialDecayTimer.new 47 | end 48 | 49 | def connection_completed 50 | @connected = true 51 | end 52 | 53 | def close_connection(*args) 54 | @is_closed = true 55 | @connected = false 56 | super(*args) 57 | end 58 | 59 | def comm_inactivity_timeout 60 | logger.info("Connection to #{@name} timed out.") 61 | schedule_reconnect 62 | end 63 | 64 | def unbind 65 | @connected = false 66 | unless @is_closed 67 | logger.info('Connection closed unintentionally.') 68 | schedule_reconnect 69 | end 70 | end 71 | 72 | def send_data(*args) 73 | super(*args) 74 | end 75 | 76 | # Override EM::Connection.receive_data to prevent it from calling 77 | # puts and randomly logging non-sense to sensu-server.log 78 | def receive_data(data) 79 | end 80 | 81 | # Reconnect normally attempts to connect at the end of the tick 82 | # Delay the reconnect for some seconds. 83 | def reconnect(time) 84 | EM.add_timer(time) do 85 | logger.info("Attempting to reconnect relay channel: #{@name}.") 86 | super(@host, @port) 87 | end 88 | end 89 | 90 | def get_reconnect_time 91 | @reconnect_timer.get_reconnect_time( 92 | @max_reconnect_time, 93 | @connection_attempt_count 94 | ) 95 | end 96 | 97 | def schedule_reconnect 98 | unless @connected 99 | @connection_attempt_count += 1 100 | reconnect_time = get_reconnect_time 101 | logger.info("Scheduling reconnect in #{@reconnect_time} seconds for relay channel: #{@name}.") 102 | reconnect(reconnect_time) 103 | end 104 | reconnect_time 105 | end 106 | 107 | def logger 108 | Sensu::Logger.get 109 | end 110 | 111 | end # RelayConnectionHandler 112 | 113 | # EndPoint 114 | # 115 | # An endpoint is a backend metric store. This is a compositional object 116 | # to help keep the rest of the code sane. 117 | class Endpoint 118 | 119 | # EM::Connection.send_data batches network connection writes in 16KB 120 | # We should start out by having all data in the queue flush in the 121 | # space of a single loop tick. 122 | MAX_QUEUE_SIZE = 16384 123 | 124 | attr_accessor :connection, :queue, :max_queue_size 125 | 126 | def initialize(name, host, port, max_queue_size = MAX_QUEUE_SIZE) 127 | @max_queue_size = max_queue_size 128 | @queue = [] 129 | @connection = EM.connect(host, port, RelayConnectionHandler) 130 | @connection.name = name 131 | @connection.host = host 132 | @connection.port = port 133 | @connection.message_queue = @queue 134 | EventMachine::PeriodicTimer.new(60) do 135 | Sensu::Logger.get.info("relay queue size for #{name}: #{queue_length} of #{max_queue_size}") 136 | end 137 | end 138 | 139 | def queue_length 140 | @queue 141 | .map(&:bytesize) 142 | .reduce(:+) || 0 143 | end 144 | 145 | def flush_to_net 146 | sent = @connection.send_data(@queue.join) 147 | @queue = [] if sent > 0 148 | end 149 | 150 | def relay_event(data) 151 | if @connection.connected 152 | @queue << data 153 | if queue_length >= @max_queue_size 154 | flush_to_net 155 | Sensu::Logger.get.debug('relay.flush_to_net: successfully flushed to network') 156 | end 157 | end 158 | end 159 | 160 | def stop 161 | if @connection.connected 162 | flush_to_net 163 | @connection.close_connection_after_writing 164 | end 165 | end 166 | 167 | end 168 | 169 | # The Relay handler expects to be called from a mutator that has prepared 170 | # output of the following format: 171 | # { 172 | # :endpoint => { :name => 'name', :host => '$host', :port => $port }, 173 | # :metric => 'formatted metric as a string' 174 | # } 175 | class Relay < Handler 176 | 177 | def initialize 178 | super 179 | @endpoints = { } 180 | @initialized = false 181 | end 182 | 183 | # ignore :reek:LongMethod 184 | def post_init 185 | @settings[:relay].keys.each do |endpoint_name| 186 | ep_name = endpoint_name.intern 187 | ep_settings = @settings[:relay][ep_name] 188 | @endpoints[ep_name] = Endpoint.new( 189 | ep_name, 190 | ep_settings['host'], 191 | ep_settings['port'], 192 | ep_settings['max_queue_size'] 193 | ) 194 | end 195 | end 196 | 197 | def definition 198 | { 199 | type: 'extension', 200 | name: 'relay', 201 | mutator: 'metrics', 202 | } 203 | end 204 | 205 | def name 206 | 'relay' 207 | end 208 | 209 | def description 210 | 'Relay metrics via a persistent TCP connection' 211 | end 212 | 213 | # ignore :reek:LongMethod 214 | def run(event_data) 215 | begin 216 | event_data.keys.each do |ep_name| 217 | logger.debug("relay.run() handling endpoint: #{ep_name}") 218 | @endpoints[ep_name].relay_event(event_data[ep_name]) unless event_data[ep_name].empty? 219 | end 220 | rescue => error 221 | yield(error.to_s, 2) 222 | end 223 | yield('', 0) 224 | end 225 | 226 | def stop 227 | @endpoints.each_value do |ep| 228 | ep.stop 229 | end 230 | yield if block_given? 231 | end 232 | 233 | def logger 234 | Sensu::Logger.get 235 | end 236 | 237 | end # Relay 238 | end # Sensu::Extension 239 | -------------------------------------------------------------------------------- /lib/sensu/extensions/mutators/metrics.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | 3 | module Sensu::Extension 4 | # Sensu::Extension::Metrics 5 | # 6 | # This mutator is meant to be used in conjuction with Sensu::Extension::Relay 7 | # It prepares metrics for relay over persistent TCP connections to metric 8 | # stores. 9 | # 10 | # Metrics sent to sensu in the following JSON format will be mutated 11 | # accordingly based on the endpoints defined in the "relay" config section. 12 | # Currently supported endpoints are Graphite and OpenTSDB. 13 | # 14 | # { name: "metric.name", value: 1, tags: { host: "hostname" } } 15 | # 16 | # Metric checks also need to specify their output type, e.g. 17 | # 18 | # output_type: "json" Or output_type: "graphite" 19 | # 20 | # Checks should also specify whether or not the hostname should be 21 | # automatically added to the formatted metric output. In the case of graphite, 22 | # the hostname is prepended to the metric name, e.g. host.fqdn.com.metric.name 23 | # In the case of OpenTSDB, the hostname is added with the "host" tag. 24 | # 25 | # DEFAULTS: 26 | # 27 | # output_type: "graphite" auto_tag_host: "yes" 28 | # 29 | # Metrics sent in graphite format will be mutated to OpenTSDB if an OpenTSDB 30 | # endpoint is defined. 31 | # 32 | # Author: Greg Poirier http://github.com/grepory and @grepory on Twitter 33 | # greg.poirier at opower.com 34 | # 35 | # Many thanks to Sean Porter, Zach Dunn, Brett Witt, and Jesse Kempf, 36 | # and Jeff Kolesky for feedback and review. 37 | class Metrics < Mutator 38 | def initialize 39 | @endpoints = {} 40 | @mutators = { 41 | graphite: method(:graphite), 42 | opentsdb: method(:opentsdb), 43 | } 44 | @event = nil 45 | end 46 | 47 | def definition 48 | { 49 | type: 'extension', 50 | name: 'metrics', 51 | } 52 | end 53 | 54 | def name 55 | 'metrics' 56 | end 57 | 58 | def description 59 | 'mutates metrics for relay to metric stores' 60 | end 61 | 62 | def run(event) 63 | @event = event 64 | logger.debug("metrics.run(): Handling event - #{event}") 65 | # The unwritten standard is graphite, if they don't specify it, assume that's 66 | # the case. 67 | event[:check][:output_type] ||= 'graphite' 68 | # We also assume that people want to auto tag their metrics. 69 | event[:check][:auto_tag_host] ||= 'yes' 70 | 71 | settings[:relay].keys.each do |endpoint_name| 72 | ep_name = endpoint_name.intern 73 | mutator = @mutators[ep_name] || next 74 | mutate(mutator, ep_name) 75 | end unless settings[:relay].nil? # keys.each 76 | # if we aren't configured we simply pass nil to the handler which it then 77 | # guards against. fail silently. 78 | yield(@endpoints, 0) 79 | end # run 80 | 81 | def mutate(mutator, ep_name) 82 | logger.debug("metrics.run mutating for #{ep_name.inspect}") 83 | check = @event[:check] 84 | output = check[:output] 85 | output_type = check[:output_type] 86 | endpoint_name = ep_name.to_s 87 | 88 | # if we receive json, we mutate based on the endpoint name 89 | if output_type == 'json' 90 | @endpoints[ep_name] = '' 91 | metrics = JSON.parse(output) 92 | if metrics.is_a?(Hash) 93 | metrics = [metrics] 94 | end 95 | metrics.each do |metric| 96 | mutated = mutator.call(metric) 97 | @endpoints[ep_name] << mutated 98 | end 99 | # don't mutate 100 | elsif output_type == endpoint_name 101 | @endpoints[ep_name] = output 102 | elsif output_type == 'graphite' && endpoint_name == 'opentsdb' 103 | @endpoints[:opentsdb] = graphite_to_opentsdb 104 | end 105 | end 106 | 107 | private 108 | 109 | def graphite(metric) 110 | out = '' 111 | out << "#{@event[:client][:name].split(/\./).reverse.join('.')}." unless @event[:check][:auto_tag_host] == 'no' 112 | out << "#{metric['name']}\t#{metric['value']}\t#{metric['timestamp']}\n" 113 | out 114 | end 115 | 116 | def opentsdb(metric) 117 | check = @event[:check] 118 | out = "put #{metric['name']} #{metric['timestamp']} #{metric['value']}" 119 | out << " check_name=#{check[:name]}" unless check[:name].nil? 120 | out << " host=#{@event[:client][:name]}" unless check[:auto_tag_host] == 'no' 121 | metric['tags'].each do |tag, value| 122 | out << " " << [tag, value].join('=') 123 | end if metric.key?('tags') 124 | out << "\n" 125 | end 126 | 127 | def graphite_to_opentsdb 128 | out = '' 129 | metrics = @event[:check][:output] 130 | client_name = @event[:client][:name] 131 | 132 | metrics.split("\n").each do |output_line| 133 | (metric_name, metric_value, epoch_time) = output_line.split 134 | # Sometimes checks outputthings we don't want or expect. 135 | # Only attempt to parse things that look like they might make sense. 136 | next unless metric_name && metric_value && epoch_time 137 | metric_value = metric_value.rstrip 138 | # attempt to strip complete hostname from the beginning, otherwise 139 | # passthrough metric name as-is 140 | metric_name = metric_name.sub(/^#{client_name}\./, '') 141 | out << "put #{metric_name} #{epoch_time} #{metric_value} host=#{client_name}\n" 142 | end 143 | out 144 | end 145 | 146 | def logger 147 | Sensu::Logger.get 148 | end 149 | 150 | end # Metrics 151 | end # Sensu::Extension 152 | -------------------------------------------------------------------------------- /lib/version.rb: -------------------------------------------------------------------------------- 1 | module Wizardvan 2 | VERSION = '0.1.0' 3 | end 4 | -------------------------------------------------------------------------------- /spec/conf.d/client.json: -------------------------------------------------------------------------------- 1 | { "client": 2 | { "name": "oppie", 3 | "address": "127.0.0.1", 4 | "environment": "production", 5 | "subscriptions": ["system"] 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /spec/conf.d/config_redis.json: -------------------------------------------------------------------------------- 1 | { 2 | "redis": { 3 | "host": "127.0.0.1", 4 | "port": 6379 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /spec/conf.d/config_relay.json: -------------------------------------------------------------------------------- 1 | { "relay": { 2 | "graphite": { "host": "127.0.0.1", "port": 31337 }, 3 | "opentsdb": { "host": "127.0.0.1", "port": 31336 } 4 | }} 5 | -------------------------------------------------------------------------------- /spec/conf.d/handler_metric_store.json: -------------------------------------------------------------------------------- 1 | { 2 | "handlers": { 3 | "metric_store": { 4 | "type": "set", 5 | "handlers": [ "relay" ] 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /spec/conf.d/rabbitmq.json: -------------------------------------------------------------------------------- 1 | { "rabbitmq": { 2 | "host": "127.0.0.1", 3 | "port": 5672, 4 | "user": "guest", 5 | "password": "guest", 6 | "vhost": "/sensu" 7 | }} 8 | -------------------------------------------------------------------------------- /spec/fixtures.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | 3 | def load_json(fixture, symbolize = true) 4 | JSON.parse(File.read(fixture), symbolize_names: symbolize) 5 | end 6 | 7 | def fixture(filename) 8 | File.expand_path("../fixtures/#{filename}", __FILE__) 9 | end 10 | 11 | module Wizardvan::Test 12 | module Fixtures 13 | 14 | BAD_GRAPHITE_EVENT = load_json(fixture('events/bad_graphite.json')) 15 | GRAPHITE_EVENT = load_json(fixture('events/graphite.json')) 16 | EMPTY_EVENT = load_json(fixture('events/empty.json')) 17 | OPENTSDB_EVENT = load_json(fixture('events/opentsdb.json')) 18 | JSON_EVENT = load_json(fixture('events/json.json')) 19 | JSON_EVENT_WITH_NAME = load_json(fixture('events/json_withname.json')) 20 | SETTINGS_FILE = fixture('settings.json') 21 | SETTINGS = load_json(SETTINGS_FILE) 22 | SETTINGS_BAD = load_json(fixture('settings_bad.json')) 23 | MUTATED_EVENT = load_json(fixture('events/mutated.json')) 24 | HOST = '127.0.0.1' 25 | PORT = 31337 26 | 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/fixtures/events/bad_graphite.json: -------------------------------------------------------------------------------- 1 | {"client":{"name":"hostname"},"check":{"output":"hostname.metric_name\tvalue\t1375226329\nbad\n"}} 2 | -------------------------------------------------------------------------------- /spec/fixtures/events/empty.json: -------------------------------------------------------------------------------- 1 | {"client":{"name":"hostname"},"check":{"output":""}} 2 | -------------------------------------------------------------------------------- /spec/fixtures/events/event.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "metric", 3 | "name": "check name", 4 | "handler": "metric_store", 5 | "status": 0, 6 | "output_type": "json", 7 | "auto_tag_host": "yes", 8 | "output": "{ 9 | \"name\": \"metric_name\", 10 | \"value\": 1, 11 | \"timestamp\": 1365624303, 12 | \"tags\": { 13 | \"service\": \"some_service\", 14 | \"some_tag\": \"some_value\" 15 | } 16 | }" 17 | } 18 | -------------------------------------------------------------------------------- /spec/fixtures/events/graphite.json: -------------------------------------------------------------------------------- 1 | {"client":{"name":"hostname"},"check":{"output":"hostname.metric_name\tvalue\t1375226329\n"}} -------------------------------------------------------------------------------- /spec/fixtures/events/json.json: -------------------------------------------------------------------------------- 1 | {"client":{"name":"hostname.example.com"},"check":{"output_type":"json","output":"{\"name\": \"metric_name\",\n \"value\": \"value\",\n \"timestamp\": 0,\n \"tags\": { \"tag\": \"tag\", \"tag2\": \"tag2\" } }\n "}} 2 | -------------------------------------------------------------------------------- /spec/fixtures/events/json_withname.json: -------------------------------------------------------------------------------- 1 | {"client":{"name":"hostname"},"check":{"output_type":"json","name":"check_name","output":"{\"name\": \"metric_name\",\n \"value\": \"value\",\n \"timestamp\": 0,\n \"tags\": { \"tag\": \"tag\", \"tag2\": \"tag2\" } }\n "}} 2 | -------------------------------------------------------------------------------- /spec/fixtures/events/mutated.json: -------------------------------------------------------------------------------- 1 | { 2 | "graphite": "mutated data", 3 | "opentsdb": "mutated data" 4 | } 5 | -------------------------------------------------------------------------------- /spec/fixtures/events/opentsdb.json: -------------------------------------------------------------------------------- 1 | {"client":{"name":"hostname"},"check":{"output_type":"opentsdb","output":"metric_name 1375226329 value\n"}} -------------------------------------------------------------------------------- /spec/fixtures/settings.json: -------------------------------------------------------------------------------- 1 | {"relay":{"graphite":{"host":"127.0.0.1","port":31337},"opentsdb":{"host":"127.0.0.1","port":31337}}} 2 | -------------------------------------------------------------------------------- /spec/fixtures/settings_bad.json: -------------------------------------------------------------------------------- 1 | {"handlers":{"relay":{"graphite":{"host":"graphite","port":1234},"opentsdb":{"host":"opentsdb","port":5678},"nonexistent":{}}}} 2 | -------------------------------------------------------------------------------- /spec/helpers.rb: -------------------------------------------------------------------------------- 1 | require 'eventmachine' 2 | 3 | module Wizardvan::Test 4 | 5 | module Helpers 6 | def timer(delay, &block) 7 | periodic_timer = EM::PeriodicTimer.new(delay) do 8 | block.call 9 | periodic_timer.cancel 10 | end 11 | end 12 | 13 | def async_wrapper(&block) 14 | EM.run do 15 | timer(10) do 16 | raise 'test timed out' 17 | end 18 | block.call 19 | end 20 | end 21 | 22 | def async_done 23 | EM.stop_event_loop 24 | end 25 | 26 | class TestServer < EM::Connection 27 | include RSpec::Matchers 28 | 29 | attr_accessor :expected 30 | 31 | def receive_data(data) 32 | EM.stop_event_loop 33 | end 34 | end 35 | end 36 | 37 | end 38 | -------------------------------------------------------------------------------- /spec/lib/sensu/extensions/handlers/relay_spec.rb: -------------------------------------------------------------------------------- 1 | require 'sensu/base' 2 | require 'sensu/extensions/handlers/relay' 3 | require 'eventmachine' 4 | 5 | describe Sensu::Extension::RelayConnectionHandler do 6 | 7 | around(:each) do |example| 8 | async_wrapper do 9 | host = Wizardvan::Test::Fixtures::HOST 10 | port = Wizardvan::Test::Fixtures::PORT 11 | EM.start_server(host, port, Wizardvan::Test::Helpers::TestServer) 12 | @connection = EM.connect(host, port, Sensu::Extension::RelayConnectionHandler) 13 | @connection.host = host 14 | @connection.port = port 15 | example.call 16 | end 17 | end 18 | 19 | it 'can connect to a metrics backend' do 20 | @connection.should be_an_instance_of(Sensu::Extension::RelayConnectionHandler) 21 | EM.next_tick do 22 | @connection.connected.should eq(true) 23 | async_done 24 | end 25 | end 26 | 27 | it 'can be closed' do 28 | @connection.close_connection 29 | @connection.connected.should eq(false) 30 | async_done 31 | end 32 | 33 | it 'can send data' do 34 | @connection.send_data('some data').should eq(9) 35 | async_done 36 | end 37 | 38 | it 'schedules a reconnect after timeout' do 39 | @connection.close_connection 40 | @connection.comm_inactivity_timeout.should be > 0 41 | async_done 42 | end 43 | 44 | it 'can schedule a reconnect' do 45 | @connection.close_connection 46 | @connection.schedule_reconnect.should be > 0 47 | async_done 48 | end 49 | 50 | it 'can reconnect' do 51 | @connection.close_connection 52 | @connection.reconnect(0) 53 | EM.next_tick do 54 | @connection.connected.should eq(true) 55 | async_done 56 | end 57 | end 58 | 59 | end 60 | 61 | describe Sensu::Extension::Endpoint do 62 | 63 | around(:each) do |example| 64 | async_wrapper do 65 | @endpoint = Sensu::Extension::Endpoint.new('name', 66 | Wizardvan::Test::Fixtures::HOST, 67 | Wizardvan::Test::Fixtures::PORT) 68 | example.run 69 | end 70 | end 71 | 72 | it 'can be created' do 73 | @endpoint.should be_an_instance_of(Sensu::Extension::Endpoint) 74 | async_done 75 | end 76 | 77 | it 'returns the correct queue length' do 78 | @endpoint.queue << 'test' 79 | @endpoint.queue << 'test' 80 | @endpoint.queue_length.should eq(8) 81 | async_done 82 | end 83 | 84 | it 'flushes the queue after every write if the queue length is 0' do 85 | @endpoint.max_queue_size = 0 86 | @endpoint.relay_event('test') 87 | @endpoint.queue_length.should eq(0) 88 | async_done 89 | end 90 | end 91 | 92 | describe ExponentialDecayTimer do 93 | 94 | MAX_RECONNECT_TIME = 500 95 | 96 | before do 97 | @timer = ExponentialDecayTimer.new 98 | end 99 | 100 | it 'returns a reconnect time below MAX_RECONNECT_TIME' do 101 | time0 = @timer.get_reconnect_time(MAX_RECONNECT_TIME, 0) 102 | time1 = @timer.get_reconnect_time(MAX_RECONNECT_TIME, 1) 103 | time0.should satisfy { |n| n > 0 } 104 | time1.should satisfy { |n| n > time0 } 105 | end 106 | 107 | it 'does not return a time > MAX_RECONNECT_TIME' do 108 | @timer.get_reconnect_time(MAX_RECONNECT_TIME, 11).should == MAX_RECONNECT_TIME 109 | end 110 | 111 | end 112 | 113 | describe Sensu::Extension::Relay do 114 | 115 | let(:mutated_event) do 116 | Wizardvan::Test::Fixtures::MUTATED_EVENT 117 | end 118 | 119 | around(:each) do |example| 120 | extensions = Sensu::Extensions.new 121 | extensions.require_directory( 122 | # bleh really? 123 | File.expand_path('../../../../../../lib/sensu/extensions', __FILE__) 124 | ) 125 | extensions.load_all 126 | @handler = extensions[:handlers]['relay'] 127 | 128 | # In the spirit of integration tests, don't just mock this... 129 | @settings = Sensu::Settings.new 130 | @settings.load_file(Wizardvan::Test::Fixtures::SETTINGS_FILE) 131 | @handler.settings = @settings.to_hash 132 | 133 | example.run 134 | end 135 | 136 | it 'can be loaded by sensu' do 137 | @handler.should be_an_instance_of(Sensu::Extension::Relay) 138 | @handler.definition[:type].should eq('extension') 139 | @handler.definition[:name].should eq('relay') 140 | end 141 | 142 | it 'can be initialized' do 143 | async_wrapper do 144 | @handler.post_init 145 | async_done 146 | end 147 | end 148 | 149 | it 'can process a mutated event' do 150 | async_wrapper do 151 | @handler.post_init 152 | @handler.run(mutated_event) do |status, error| 153 | status.should eq('') 154 | error.should eq(0) 155 | async_done 156 | end 157 | end 158 | end 159 | 160 | end 161 | -------------------------------------------------------------------------------- /spec/lib/sensu/extensions/mutators/metrics_spec.rb: -------------------------------------------------------------------------------- 1 | require 'sensu/base' 2 | 3 | describe 'Sensu::Extension::Metrics' do 4 | 5 | let(:graphite_event) do 6 | Wizardvan::Test::Fixtures::GRAPHITE_EVENT 7 | end 8 | 9 | let(:bad_graphite_event) do 10 | Wizardvan::Test::Fixtures::BAD_GRAPHITE_EVENT 11 | end 12 | 13 | let(:opentsdb_event) do 14 | Wizardvan::Test::Fixtures::OPENTSDB_EVENT 15 | end 16 | 17 | let(:json_event) do 18 | Wizardvan::Test::Fixtures::JSON_EVENT 19 | end 20 | 21 | let(:json_event_with_name) do 22 | Wizardvan::Test::Fixtures::JSON_EVENT_WITH_NAME 23 | end 24 | 25 | let(:settings) do 26 | Wizardvan::Test::Fixtures::SETTINGS 27 | end 28 | 29 | let(:settings_bad) do 30 | Wizardvan::Test::Fixtures::SETTINGS_BAD 31 | end 32 | 33 | let(:empty_event) do 34 | Wizardvan::Test::Fixtures::EMPTY_EVENT 35 | end 36 | 37 | before(:all) do 38 | extensions = Sensu::Extensions.new 39 | extensions.require_directory( 40 | # bleh really? 41 | File.expand_path('../../../../../../lib/sensu/extensions', __FILE__) 42 | ) 43 | extensions.load_all 44 | @mutator = extensions[:mutators]['metrics'] 45 | end 46 | 47 | before(:each) do 48 | @mutator.settings = settings 49 | end 50 | 51 | it 'can be loaded by sensu' do 52 | @mutator.should be_an_instance_of(Sensu::Extension::Metrics) 53 | @mutator.definition[:type].should eq('extension') 54 | @mutator.definition[:name].should eq('metrics') 55 | end 56 | 57 | it 'successfully returns endpoints hash when configured' do 58 | @mutator.run(graphite_event) do |output, status| 59 | output.should be_an_instance_of(Hash) 60 | end 61 | end 62 | 63 | it 'does nothing for empty metrics' do 64 | @mutator.run(empty_event) do |output, status| 65 | status.should == 0 66 | output[:graphite].should == "" 67 | output[:opentsdb].should == "" 68 | end 69 | end 70 | 71 | it 'passes metrics through for graphite' do 72 | @mutator.run(graphite_event) do |output, status| 73 | output[:graphite].should == graphite_event[:check][:output] 74 | end 75 | end 76 | 77 | it 'should ignore malformed lines in graphite events when mutating for opentsdb' do 78 | @mutator.run(bad_graphite_event) do |output, status| 79 | output[:opentsdb].should_not match(/bad/) 80 | end 81 | end 82 | 83 | it 'passes metrics through for opentsdb' do 84 | @mutator.run(opentsdb_event) do |output, status| 85 | output[:opentsdb].should == opentsdb_event[:check][:output] 86 | end 87 | end 88 | 89 | it 'strips hostnames from opentsdb metric names' do 90 | @mutator.run(graphite_event) do |output, status| 91 | output[:opentsdb].should =~ /^put metric_name/ 92 | end 93 | end 94 | 95 | it 'automatically tags mutated metrics for opentsdb with host' do 96 | @mutator.run(graphite_event) do |output, status| 97 | output[:opentsdb].should =~ /host=hostname\n$/ 98 | end 99 | end 100 | 101 | it 'mutates json to graphite' do 102 | @mutator.run(json_event) do |output, status| 103 | output[:graphite].should == "com.example.hostname.metric_name\tvalue\t0\n" 104 | end 105 | end 106 | 107 | it 'mutates json to opentsdb' do 108 | @mutator.run(json_event) do |output, status| 109 | output[:opentsdb].should == "put metric_name 0 value host=hostname.example.com tag=tag tag2=tag2\n" 110 | end 111 | end 112 | 113 | it 'mutates json to opentsdb and includes name if it exists' do 114 | @mutator.run(json_event_with_name) do |output, status| 115 | output[:opentsdb].should == "put metric_name 0 value check_name=check_name host=hostname tag=tag tag2=tag2\n" 116 | end 117 | end 118 | 119 | it 'handles non-existent mutators gracefully-ish' do 120 | @mutator.settings = settings_bad 121 | @mutator.run(json_event) do |output, status| 122 | status.should == 0 123 | end 124 | end 125 | 126 | it 'does not require tags' do 127 | @mutator.run(opentsdb_event) do |output, status| 128 | output[:opentsdb].should == opentsdb_event[:check][:output] 129 | end 130 | end 131 | end 132 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'coveralls' 2 | require 'simplecov' 3 | 4 | $LOAD_PATH << File.expand_path('../../lib', __FILE__) 5 | 6 | require File.expand_path('../helpers.rb', __FILE__) 7 | require File.expand_path('../fixtures.rb', __FILE__) 8 | 9 | include Wizardvan::Test::Helpers 10 | 11 | SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[ 12 | SimpleCov::Formatter::HTMLFormatter, 13 | Coveralls::SimpleCov::Formatter 14 | ] 15 | SimpleCov.start 16 | -------------------------------------------------------------------------------- /wizardvan.gemspec: -------------------------------------------------------------------------------- 1 | require File.expand_path('../lib/version', __FILE__) 2 | 3 | Gem::Specification.new do |gem| 4 | gem.authors = ['Greg Poirer'] 5 | gem.email = ['greg.poirier@opower.com'] 6 | gem.description = <<-DESCRIPTION 7 | Wizadvan: A Sensu Metrics Relay - Establishes persistent TCP connections to multiple 8 | metrics backends and relays metrics to them. Optionally, Wizardvan can mutate from 9 | a common, unified JSON metric format to any number of formats expected by backend 10 | metric stores. 11 | DESCRIPTION 12 | gem.summary = 'Sensu Metrics Relay' 13 | gem.files = `git ls-files`.split($\) 14 | gem.executables = [] 15 | gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) 16 | gem.name = 'wizardvan' 17 | gem.require_paths = ['lib'] 18 | gem.version = Wizardvan::VERSION 19 | 20 | gem.add_dependency('sensu', '0.12.1') 21 | 22 | # development dependencies 23 | gem.add_development_dependency('rspec', '~> 2.13.0') 24 | gem.add_development_dependency('rake', '~> 10.1.0') 25 | gem.add_development_dependency('simplecov', '~> 0.7.0') 26 | gem.add_development_dependency('coveralls', '~> 0.6.7') 27 | gem.add_development_dependency('guard', '~> 1.8.0') 28 | gem.add_development_dependency('guard-rspec', '~> 3.0.1') 29 | gem.add_development_dependency('rubocop', '~> 0.8.3') 30 | gem.add_development_dependency('guard-rubocop', '~> 0.0.4') 31 | gem.add_development_dependency('metric_fu', '~> 4.2.0') 32 | gem.add_development_dependency('guard-reek', '~> 0.0.4') 33 | end 34 | --------------------------------------------------------------------------------