├── .rspec ├── .gitignore ├── Procfile ├── vendor └── cache │ ├── json-1.8.0.gem │ ├── rack-1.5.2.gem │ ├── thin-1.3.1.gem │ ├── tilt-1.3.3.gem │ ├── daemons-1.1.8.gem │ ├── dotenv-0.9.0.gem │ ├── rake-0.9.2.2.gem │ ├── rspec-2.11.0.gem │ ├── sinatra-1.3.2.gem │ ├── thor-0.18.1.gem │ ├── diff-lcs-1.1.3.gem │ ├── foreman-0.63.0.gem │ ├── rack-test-0.6.1.gem │ ├── yajl-ruby-1.1.0.gem │ ├── rspec-core-2.11.1.gem │ ├── rspec-mocks-2.11.2.gem │ ├── eventmachine-0.12.10.gem │ ├── newrelic_rpm-3.6.5.130.gem │ ├── rack-protection-1.2.0.gem │ └── rspec-expectations-2.11.2.gem ├── Rakefile ├── config.ru ├── lib ├── backstop.rb └── backstop │ ├── collectd │ ├── plugins │ │ ├── swap.rb │ │ ├── droid.rb │ │ ├── conntrack.rb │ │ ├── memory.rb │ │ ├── cpu.rb │ │ ├── nfsiostat.rb │ │ ├── tcpconns.rb │ │ ├── fsperformance.rb │ │ ├── df.rb │ │ ├── load.rb │ │ ├── disk.rb │ │ ├── interface.rb │ │ └── processes.rb │ └── parser.rb │ ├── config.rb │ ├── publisher.rb │ └── web.rb ├── Gemfile ├── spec ├── spec_helper.rb └── backstop │ ├── bad_collectd_data.json │ ├── good_collectd_data.json │ ├── bad_github_data.json │ ├── good_github_data.json │ ├── publisher_spec.rb │ └── web_spec.rb ├── config └── newrelic.yml ├── Gemfile.lock ├── LICENSE └── README.md /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format progress 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle 2 | .env 3 | .foreman 4 | 5 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: bundle exec rackup -p $PORT -s thin 2 | -------------------------------------------------------------------------------- /vendor/cache/json-1.8.0.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/json-1.8.0.gem -------------------------------------------------------------------------------- /vendor/cache/rack-1.5.2.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/rack-1.5.2.gem -------------------------------------------------------------------------------- /vendor/cache/thin-1.3.1.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/thin-1.3.1.gem -------------------------------------------------------------------------------- /vendor/cache/tilt-1.3.3.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/tilt-1.3.3.gem -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rspec/core/rake_task' 2 | 3 | RSpec::Core::RakeTask.new(:spec) 4 | task :default => :spec 5 | -------------------------------------------------------------------------------- /vendor/cache/daemons-1.1.8.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/daemons-1.1.8.gem -------------------------------------------------------------------------------- /vendor/cache/dotenv-0.9.0.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/dotenv-0.9.0.gem -------------------------------------------------------------------------------- /vendor/cache/rake-0.9.2.2.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/rake-0.9.2.2.gem -------------------------------------------------------------------------------- /vendor/cache/rspec-2.11.0.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/rspec-2.11.0.gem -------------------------------------------------------------------------------- /vendor/cache/sinatra-1.3.2.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/sinatra-1.3.2.gem -------------------------------------------------------------------------------- /vendor/cache/thor-0.18.1.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/thor-0.18.1.gem -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | $:.unshift File.dirname(__FILE__) + '/lib' 2 | require 'backstop/web' 3 | 4 | run Backstop::Application 5 | -------------------------------------------------------------------------------- /vendor/cache/diff-lcs-1.1.3.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/diff-lcs-1.1.3.gem -------------------------------------------------------------------------------- /vendor/cache/foreman-0.63.0.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/foreman-0.63.0.gem -------------------------------------------------------------------------------- /vendor/cache/rack-test-0.6.1.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/rack-test-0.6.1.gem -------------------------------------------------------------------------------- /vendor/cache/yajl-ruby-1.1.0.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/yajl-ruby-1.1.0.gem -------------------------------------------------------------------------------- /vendor/cache/rspec-core-2.11.1.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/rspec-core-2.11.1.gem -------------------------------------------------------------------------------- /vendor/cache/rspec-mocks-2.11.2.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/rspec-mocks-2.11.2.gem -------------------------------------------------------------------------------- /vendor/cache/eventmachine-0.12.10.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/eventmachine-0.12.10.gem -------------------------------------------------------------------------------- /vendor/cache/newrelic_rpm-3.6.5.130.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/newrelic_rpm-3.6.5.130.gem -------------------------------------------------------------------------------- /vendor/cache/rack-protection-1.2.0.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/rack-protection-1.2.0.gem -------------------------------------------------------------------------------- /vendor/cache/rspec-expectations-2.11.2.gem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/github/backstop/master/vendor/cache/rspec-expectations-2.11.2.gem -------------------------------------------------------------------------------- /lib/backstop.rb: -------------------------------------------------------------------------------- 1 | require 'socket' 2 | require 'uri' 3 | 4 | require 'backstop/config' 5 | require 'backstop/collectd/parser' 6 | require 'backstop/publisher' 7 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/swap.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # swap stats 3 | def parse_plugin_swap 4 | [{ 5 | metric: "swap.#{data['type_instance']}", 6 | value: data['values'][0] 7 | }] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/droid.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # droid stats 3 | def parse_plugin_droid 4 | [{ 5 | metric: "droid.#{data['type_instance']}", 6 | value: data['values'][0] 7 | }] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/conntrack.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # conntrack stats 3 | def parse_plugin_conntrack 4 | [{ 5 | metric: 'conntrack.connections', 6 | value: data['values'][0] 7 | }] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/memory.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # memory stats 3 | def parse_plugin_memory 4 | [{ 5 | metric: "memory.#{data['type_instance']}", 6 | value: data['values'][0] 7 | }] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/cpu.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # cpu stats 3 | def parse_plugin_cpu 4 | [{ 5 | metric: "cpu.#{data['plugin_instance']}.#{data['type_instance']}", 6 | value: data['values'][0] 7 | }] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/nfsiostat.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # nfs iostat 3 | def parse_plugin_nfsiostat 4 | [{ 5 | metric: "#{data['plugin']}.#{data['plugin_instance']}.#{data['type_instance']}", 6 | value: data['values'][0] 7 | }] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/tcpconns.rb: -------------------------------------------------------------------------------- 1 | #class CollectdData 2 | # # tcpconns stats 3 | # def parse_plugin_tcpconns 4 | # [{ 5 | # metric: "tcpconns.#{data['plugin_instance']}.#{data['type_instance']}", 6 | # value: data['values'][0] 7 | # }] 8 | # end 9 | #end 10 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/fsperformance.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # file system performance 3 | def parse_plugin_fsperformance 4 | [{ 5 | metric: "#{data['plugin']}.#{data['plugin_instance']}.#{data['type_instance']}", 6 | value: data['values'][0] 7 | }] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | ruby '1.9.3' 4 | 5 | gem 'bundler' 6 | gem 'foreman' 7 | gem 'dotenv' 8 | gem 'sinatra' 9 | gem 'thin' 10 | gem 'json' 11 | gem 'yajl-ruby' 12 | gem 'newrelic_rpm' 13 | 14 | group :test do 15 | gem 'rake' 16 | gem 'rspec' 17 | gem 'rack-test' 18 | end 19 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/df.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # disk partition stats 3 | def parse_plugin_df 4 | [ 5 | { metric: "df.#{data['type_instance']}.used", value: data['values'][0] }, 6 | { metric: "df.#{data['type_instance']}.free", value: data['values'][1] } 7 | ] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/load.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # system load 3 | def parse_plugin_load 4 | [ 5 | { metric: 'load.1m', value: data['values'][0] }, 6 | { metric: 'load.5m', value: data['values'][1] }, 7 | { metric: 'load.15m', value: data['values'][2] } 8 | ] 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/disk.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # disk volume stats 3 | def parse_plugin_disk 4 | [ 5 | { metric: "disk.#{data['plugin_instance']}.#{data['type']}.write", value: data['values'][0] }, 6 | { metric: "disk.#{data['plugin_instance']}.#{data['type']}.read", value: data['values'][1] } 7 | ] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/interface.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | # interface stats 3 | def parse_plugin_interface 4 | [ 5 | { metric: "net.#{data['type_instance']}.#{data['type']}.in", value: data['values'][0] }, 6 | { metric: "net.#{data['type_instance']}.#{data['type']}.out",value: data['values'][1] } 7 | ] 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'backstop' 2 | 3 | ENV['CARBON_URLS'] = 'carbon://1.1.1.1:5000' 4 | ENV['PREFIXES'] = 'test' 5 | 6 | RSpec.configure do |config| 7 | config.treat_symbols_as_metadata_keys_with_true_values = true 8 | config.run_all_when_everything_filtered = true 9 | config.filter_run :focus 10 | config.order = 'random' 11 | end 12 | -------------------------------------------------------------------------------- /spec/backstop/bad_collectd_data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "values": [1901474177], 4 | "dstypes": ["counter"], 5 | "dsnames": ["value"], 6 | "interval": 10, 7 | "host": "leeloo.octo.it", 8 | "plugin": "cpu", 9 | "plugin_instance": "0", 10 | "type": "cpu", 11 | "type_instance": "idle" 12 | } 13 | ] 14 | -------------------------------------------------------------------------------- /spec/backstop/good_collectd_data.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "values": [1901474177], 4 | "dstypes": ["counter"], 5 | "dsnames": ["value"], 6 | "time": 1280959128, 7 | "interval": 10, 8 | "host": "leeloo.octo.it", 9 | "plugin": "cpu", 10 | "plugin_instance": "0", 11 | "type": "cpu", 12 | "type_instance": "idle" 13 | } 14 | ] 15 | -------------------------------------------------------------------------------- /lib/backstop/config.rb: -------------------------------------------------------------------------------- 1 | module Backstop 2 | module Config 3 | def self.env!(key) 4 | ENV[key] || raise("missing #{key}") 5 | end 6 | 7 | def self.deploy; env!('DEPLOY'); end 8 | def self.port; env!('PORT').to_i; end 9 | def self.carbon_urls; env!('CARBON_URLS').split(','); end 10 | def self.prefixes; env!('PREFIXES').split(','); end 11 | def self.api_key; ENV['API_KEY']; end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /config/newrelic.yml: -------------------------------------------------------------------------------- 1 | --- 2 | production: 3 | agent_enabled: true 4 | error_collector: 5 | capture_source: true 6 | enabled: true 7 | ignore_errors: ActionController::RoutingError 8 | apdex_t: 0.5 9 | ssl: true 10 | monitor_mode: true 11 | license_key: <%= ENV['NEW_RELIC_LICENSE_KEY'] %> 12 | developer_mode: false 13 | app_name: <%= ENV['NEW_RELIC_APP_NAME'] %> 14 | capture_params: false 15 | log_level: info 16 | -------------------------------------------------------------------------------- /lib/backstop/publisher.rb: -------------------------------------------------------------------------------- 1 | module Backstop 2 | class Publisher 3 | attr_reader :connections, :api_key 4 | 5 | def initialize(urls, opts={}) 6 | @connections = [] 7 | @connections = urls.map { |u| URI.parse(u) }.map { |u| TCPSocket.new(u.host, u.port) } 8 | @api_key = opts[:api_key] 9 | end 10 | 11 | def close_all 12 | connections.each { |c| c.close } 13 | end 14 | 15 | def metric_name(name) 16 | api_key ? "#{api_key}.#{name}" : name 17 | end 18 | 19 | def publish(name, value, time=Time.now.to_i) 20 | begin 21 | connections.sample.puts("#{metric_name(name)} #{value} #{time}") 22 | rescue Errno::EPIPE => e 23 | raise e 24 | end 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /spec/backstop/bad_github_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d", 3 | "commits": [ 4 | { 5 | "id": "41a212ee83ca127e3c8cf465891ab7216a705f59", 6 | "url": "http://github.com/defunkt/github/commit/41a212ee83ca127e3c8cf465891ab7216a705f59", 7 | "author": { 8 | "email": "chris@ozmm.org", 9 | "name": "Chris Wanstrath" 10 | }, 11 | "message": "okay i give in", 12 | "timestamp": "2008-02-15T14:57:17-08:00", 13 | "added": ["filepath.rb"] 14 | }, 15 | { 16 | "id": "de8251ff97ee194a289832576287d6f8ad74e3d0", 17 | "url": "http://github.com/defunkt/github/commit/de8251ff97ee194a289832576287d6f8ad74e3d0", 18 | "author": { 19 | "email": "chris@ozmm.org", 20 | "name": "Chris Wanstrath" 21 | }, 22 | "message": "update pricing a tad", 23 | "timestamp": "2008-02-15T14:36:34-08:00" 24 | } 25 | ], 26 | "after": "de8251ff97ee194a289832576287d6f8ad74e3d0", 27 | "ref": "refs/heads/master" 28 | } 29 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | daemons (1.1.8) 5 | diff-lcs (1.1.3) 6 | dotenv (0.9.0) 7 | eventmachine (0.12.10) 8 | foreman (0.63.0) 9 | dotenv (>= 0.7) 10 | thor (>= 0.13.6) 11 | json (1.8.0) 12 | newrelic_rpm (3.6.5.130) 13 | rack (1.5.2) 14 | rack-protection (1.2.0) 15 | rack 16 | rack-test (0.6.1) 17 | rack (>= 1.0) 18 | rake (0.9.2.2) 19 | rspec (2.11.0) 20 | rspec-core (~> 2.11.0) 21 | rspec-expectations (~> 2.11.0) 22 | rspec-mocks (~> 2.11.0) 23 | rspec-core (2.11.1) 24 | rspec-expectations (2.11.2) 25 | diff-lcs (~> 1.1.3) 26 | rspec-mocks (2.11.2) 27 | sinatra (1.3.2) 28 | rack (~> 1.3, >= 1.3.6) 29 | rack-protection (~> 1.2) 30 | tilt (~> 1.3, >= 1.3.3) 31 | thin (1.3.1) 32 | daemons (>= 1.0.9) 33 | eventmachine (>= 0.12.6) 34 | rack (>= 1.0.0) 35 | thor (0.18.1) 36 | tilt (1.3.3) 37 | yajl-ruby (1.1.0) 38 | 39 | PLATFORMS 40 | ruby 41 | 42 | DEPENDENCIES 43 | bundler 44 | dotenv 45 | foreman 46 | json 47 | newrelic_rpm 48 | rack-test 49 | rake 50 | rspec 51 | sinatra 52 | thin 53 | yajl-ruby 54 | -------------------------------------------------------------------------------- /spec/backstop/good_github_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d", 3 | "repository": { 4 | "url": "http://github.com/defunkt/github", 5 | "name": "github", 6 | "description": "You're lookin' at it.", 7 | "watchers": 5, 8 | "forks": 2, 9 | "private": 1, 10 | "owner": { 11 | "email": "chris@ozmm.org", 12 | "name": "defunkt" 13 | } 14 | }, 15 | "commits": [ 16 | { 17 | "id": "41a212ee83ca127e3c8cf465891ab7216a705f59", 18 | "url": "http://github.com/defunkt/github/commit/41a212ee83ca127e3c8cf465891ab7216a705f59", 19 | "author": { 20 | "email": "chris@ozmm.org", 21 | "name": "Chris Wanstrath" 22 | }, 23 | "message": "okay i give in", 24 | "timestamp": "2008-02-15T14:57:17-08:00", 25 | "added": ["filepath.rb"] 26 | }, 27 | { 28 | "id": "de8251ff97ee194a289832576287d6f8ad74e3d0", 29 | "url": "http://github.com/defunkt/github/commit/de8251ff97ee194a289832576287d6f8ad74e3d0", 30 | "author": { 31 | "email": "chris@ozmm.org", 32 | "name": "Chris Wanstrath" 33 | }, 34 | "message": "update pricing a tad", 35 | "timestamp": "2008-02-15T14:36:34-08:00" 36 | } 37 | ], 38 | "after": "de8251ff97ee194a289832576287d6f8ad74e3d0", 39 | "ref": "refs/heads/master" 40 | } 41 | -------------------------------------------------------------------------------- /lib/backstop/collectd/parser.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | 3 | class CollectdData 4 | 5 | # ALL PLUGIN CHECKS ARE EXPECTED TO RETURN AN ARRAY OF HASHES OR AN EMPTY ARRAY 6 | Dir[File.dirname(__FILE__) + '/plugins/*.rb'].each do |file| 7 | f = File.basename(file).gsub(/\.rb/, '') 8 | require "backstop/collectd/plugins/#{f}" 9 | end 10 | 11 | attr_accessor :data 12 | 13 | def initialize(data) 14 | self.data = data 15 | end 16 | 17 | def parse 18 | base = parse_base 19 | plugin = parse_plugin 20 | plugin.map {|p| p.merge base} 21 | end 22 | 23 | # extract cloud, slot, and id 24 | def parse_base 25 | hostname = data['host'].gsub('DOT','.').gsub('DASH', '-') 26 | parts = hostname.split('.') 27 | id = parts.last 28 | slot = parts[-2] 29 | cloud = parts.first(parts.size-2).join('.') 30 | measure_period = (data['interval'] || 10).to_i 31 | {id: id, slot: slot, cloud: cloud, measure_period: measure_period, measure_time: data['time']} 32 | end 33 | 34 | # extract the juicy bits, but do it dynamically 35 | # we check for the existence of a predefined method called parse_plugin_PLUGIN 36 | # if it exists, we dispatch. If it doesn't, we return an empty array 37 | def parse_plugin 38 | plugin = data['plugin'] 39 | method = "parse_plugin_#{plugin}".to_sym 40 | if self.respond_to? method 41 | send(method) 42 | else 43 | [] 44 | end 45 | end 46 | 47 | end 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Jason Dixon 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * The name Jason Dixon may not be used to endorse or promote products 15 | derived from this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL JASON DIXON BE LIABLE FOR ANY DIRECT, INDIRECT, 21 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 23 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 24 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 25 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 26 | EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /lib/backstop/collectd/plugins/processes.rb: -------------------------------------------------------------------------------- 1 | class CollectdData 2 | def parse_plugin_processes 3 | # matches specific proceses 4 | if !data['plugin_instance'].empty? 5 | ps_value_map = { 6 | 'ps_count' => ['num_proc', 'num_thread'], 7 | 'ps_disk_ops' => ['read', 'write'], 8 | 'ps_disk_octets' => ['read', 'write'], 9 | 'ps_pagefaults' => ['minor', 'major'], 10 | 'ps_cputime' => ['user', 'system'] 11 | } 12 | 13 | if (map = ps_value_map[data['type']]) 14 | [ 15 | { 16 | metric: "#{data['plugin']}.#{data['plugin_instance']}.#{data['type']}.#{map[0]}", 17 | value: data['values'][0] 18 | }, 19 | { 20 | metric: "#{data['plugin']}.#{data['plugin_instance']}.#{data['type']}.#{map[1]}", 21 | value: data['values'][1] 22 | } 23 | ] 24 | else 25 | [ 26 | { 27 | metric: "#{data['plugin']}.#{data['plugin_instance']}.#{data['type']}", 28 | value: data['values'][0] 29 | } 30 | ] 31 | end 32 | elsif data['type_instance'].empty? 33 | # matches fork_rate 34 | [ 35 | { 36 | metric: "processes.#{data['type']}", 37 | value: data['values'][0] 38 | } 39 | ] 40 | else 41 | # everything else in ps_state 42 | [ 43 | { 44 | metric: "processes.#{data['type_instance']}", 45 | value: data['values'][0] 46 | } 47 | ] 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /spec/backstop/publisher_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Backstop::Publisher do 4 | it 'should initialize with an array of urls' do 5 | urls = ['tcp://10.0.0.1:5000', 'tcp://10.0.0.1:5001'] 6 | TCPSocket.should_receive(:new).with('10.0.0.1', 5000) 7 | TCPSocket.should_receive(:new).with('10.0.0.1', 5001) 8 | b = Backstop::Publisher.new(urls) 9 | b.connections.count.should eq 2 10 | end 11 | 12 | it 'should publish data' do 13 | urls = ['tcp://10.0.0.1:5000'] 14 | socket_double = double('TCPSocket') 15 | TCPSocket.should_receive(:new).with('10.0.0.1', 5000) { socket_double } 16 | b = Backstop::Publisher.new(urls) 17 | 18 | socket_double.should_receive(:puts).with('foo 1 1') 19 | b.publish('foo', 1, 1) 20 | end 21 | 22 | it 'should include a timestamp if you do not provide one' do 23 | urls = ['tcp://10.0.0.1:5000'] 24 | socket_double = double('TCPSocket') 25 | TCPSocket.should_receive(:new).with('10.0.0.1', 5000) { socket_double } 26 | b = Backstop::Publisher.new(urls) 27 | 28 | Time.should_receive(:now) { 12345 } 29 | socket_double.should_receive(:puts).with('foo 1 12345') 30 | b.publish('foo', 1) 31 | end 32 | 33 | it 'should apply an api key if you provide one' do 34 | urls = ['tcp://10.0.0.1:5000'] 35 | socket_double = double('TCPSocket') 36 | TCPSocket.should_receive(:new).with('10.0.0.1', 5000) { socket_double } 37 | b = Backstop::Publisher.new(urls, :api_key => '12345') 38 | 39 | socket_double.should_receive(:puts).with('12345.foo 1 1') 40 | b.publish('foo', 1, 1) 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /spec/backstop/web_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require 'backstop/web' 4 | require 'rack/test' 5 | 6 | describe Backstop::Application do 7 | include Rack::Test::Methods 8 | 9 | def app 10 | Backstop::Application 11 | end 12 | 13 | before(:each) do 14 | app.class_variable_set :@@publisher, nil 15 | end 16 | 17 | context 'GET /health' do 18 | it 'should handle GET /health' do 19 | get '/health' 20 | last_response.should be_ok 21 | end 22 | end 23 | 24 | context 'POST /publish/:name' do 25 | it 'should require JSON' do 26 | post '/publish/foo', 'foo' 27 | last_response.should_not be_ok 28 | last_response.status.should eq(400) 29 | end 30 | 31 | it 'should handle a single metric' do 32 | p = double('publisher') 33 | Backstop::Publisher.should_receive(:new) { p } 34 | p.should_receive(:publish).with('test.bar', 12345, 1) 35 | post '/publish/test', { :metric => 'bar', :value => 12345, :measure_time => 1 }.to_json 36 | last_response.should be_ok 37 | end 38 | 39 | it 'should handle an array of metrics' do 40 | p = double('publisher') 41 | Backstop::Publisher.should_receive(:new) { p } 42 | p.should_receive(:publish).with('test.bar', 12345, 1) 43 | p.should_receive(:publish).with('test.bar', 12344, 2) 44 | post '/publish/test', [{ :metric => 'bar', :value => 12345, :measure_time => 1 }, { :metric => 'bar', :value => 12344, :measure_time => 2} ].to_json 45 | last_response.should be_ok 46 | end 47 | end 48 | 49 | context 'POST /collectd' do 50 | let(:good_collectd_data) { File.open(File.dirname(__FILE__) + '/good_collectd_data.json').read } 51 | let(:bad_collectd_data) { File.open(File.dirname(__FILE__) + '/bad_collectd_data.json').read } 52 | 53 | it 'should require JSON' do 54 | post '/collectd', 'foo' 55 | last_response.should_not be_ok 56 | last_response.status.should eq(400) 57 | end 58 | 59 | it 'should handle a collectd metric' do 60 | p = double('publisher') 61 | Backstop::Publisher.should_receive(:new) { p } 62 | p.should_receive(:publish).with('mitt.leeloo.octo.it.cpu.0.idle', 1901474177, 1280959128) 63 | post '/collectd', good_collectd_data 64 | last_response.body.should eq('ok') 65 | last_response.status.should eq(200) 66 | end 67 | 68 | it 'should complain if missing fields' do 69 | post '/collectd', bad_collectd_data 70 | last_response.status.should eq(400) 71 | last_response.body.should eq('missing fields') 72 | end 73 | end 74 | 75 | context 'POST /github' do 76 | let(:good_github_data) { File.open(File.dirname(__FILE__) + '/good_github_data.json').read } 77 | let(:bad_github_data) { File.open(File.dirname(__FILE__) + '/bad_github_data.json').read } 78 | 79 | it 'should require JSON' do 80 | post '/github', { :payload => 'foo' } 81 | last_response.should_not be_ok 82 | last_response.status.should eq(400) 83 | end 84 | 85 | it 'should take a github push' do 86 | p = double('publisher') 87 | Backstop::Publisher.should_receive(:new) { p } 88 | p.should_receive(:publish).with('github.github.refs.heads.master.chris-ozmm-org.de8251ff97ee194a289832576287d6f8ad74e3d0', 1, '1203114994') 89 | p.should_receive(:publish).with('github.github.refs.heads.master.chris-ozmm-org.41a212ee83ca127e3c8cf465891ab7216a705f59', 1, '1203116237') 90 | post '/github', { :payload => good_github_data } 91 | last_response.should be_ok 92 | end 93 | 94 | it 'should complain if missing fields' do 95 | post '/github', { :payload => bad_github_data } 96 | last_response.should_not be_ok 97 | end 98 | end 99 | end 100 | 101 | 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Backstop 2 | 3 | [![Build Status](https://secure.travis-ci.org/obfuscurity/backstop.png?branch=master)](http://travis-ci.org/obfuscurity/backstop) 4 | 5 | Backstop is a simple endpoint for submitting metrics to Graphite. It accepts JSON data via HTTP POST and proxies the data to one or more Carbon/Graphite listeners. 6 | 7 | ## Usage 8 | 9 | ### Collectd Metrics 10 | 11 | Backstop supports submission of metrics via the Collectd [write_http](http://collectd.org/wiki/index.php/Plugin:Write_HTTP) output plugin. A sample client configuration: 12 | 13 | ``` 14 | 15 | 16 | Format "JSON" 17 | User "" 18 | Password "" 19 | 20 | 21 | ``` 22 | 23 | ### GitHub Post-Receive Hooks 24 | 25 | Backstop can receive commit data from GitHub [post-receive webhooks](https://help.github.com/articles/post-receive-hooks). Your WebHook URL should consist of the Backstop service URL with the `/github` endpoint. For example, `https://backstop.example.com/github`. 26 | 27 | All GitHub commit metrics contain the project name, branch information, author email and commit identifier, and are stored with a value of `1`. These can then be visualized as annotation-style metrics using Graphite's `drawAsInfinite()` function. Sample metric: 28 | 29 | ``` 30 | github.project.refs.heads.master.bob-example-com.10af2cb02eadd4cb1a3e43aa9cae47ef2cd07016 1 1203116237 31 | ``` 32 | 33 | ### PagerDuty Incident Webhooks 34 | 35 | Backstop can also receive PagerDuty incidents courtesy of Jesse Newland's [pagerduty-incident-webhooks](https://github.com/github/pagerduty-incident-webhooks) project. When deploying `pagerduty-incident-webhooks` make sure to set `PAGERDUTY_WEBHOOK_ENDPOINT` to your Backstop service URL with the `/pagerduty` endpoint. For example, `https://backstop.example.com/pagerduty`. 36 | 37 | Metrics will be stored under the `alerts` prefix with a value of `1`. These can then be visualized as annotation-style metrics using Graphite's `drawAsInfinite()` function. Sample metric: 38 | 39 | ``` 40 | alerts.nagios.web1.diskspace 1 1365206103 41 | ``` 42 | 43 | ### Custom Metrics 44 | 45 | Use the `/publish` endpoint in conjunction with one of the approved `PREFIXES` for submitting metrics to Backstop. In most environments it makes sense to use distinct prefixes for normal (e.g. gauge, counters, etc) metrics vs annotation (event-style) metrics. `PREFIXES` is defined as a comma-delimited list of prefix strings. For example: 46 | 47 | ```bash 48 | export PREFIXES='test,app1,app2' 49 | ``` 50 | 51 | #### Sending Metrics 52 | 53 | Here is a basic example for posting an application metric to the `custom` prefix. 54 | 55 | ```ruby 56 | RestClient.post("https://backstop.example.com/publish/custom", 57 | [{:metric => key, :value => value, :measure_time => Time.now.to_i}].to_json) 58 | ``` 59 | 60 | #### Sending Annotations 61 | 62 | Here is an example for posting a software release announcement to the `note` prefix. 63 | 64 | ```ruby 65 | RestClient.post("https://backstop.example.com/publish/note", 66 | [{:metric => "foobar.release", :value => "v214", :measure_time => Time.now.to_i}].to_json) 67 | ``` 68 | 69 | #### Using with Hosted Graphite 70 | 71 | Graphite hosting service [Hosted Graphite](https://www.hostedgraphite.com) requires metrics to be submitted with an API key prepended to the metric. To use their service, just define the `API_KEY` environment variable. 72 | 73 | ## Deployment 74 | 75 | Backstop supports optional Basic Authentication through Rack::Auth::Basic. Simply set BACKSTOP_AUTH to your colon-delimited credentials (e.g. `user:pass`). 76 | 77 | The `CARBON_URLS` variable must be set to one or more valid destinations. Examples: 78 | 79 | ``` 80 | export CARBON_URLS="carbon://10.10.10.10:2003,carbon://10.10.20.10:2003" 81 | ``` 82 | 83 | ### Local 84 | 85 | The following instructions assume a working Ruby installation with the bundler gem already installed on your system. 86 | 87 | ```bash 88 | $ git clone https://github.com/obfuscurity/backstop.git 89 | $ cd backstop 90 | $ bundle install 91 | $ export CARBON_URLS=... 92 | $ export PREFIXES=... 93 | $ export BACKSTOP_AUTH=... (optional) 94 | $ foreman start 95 | ``` 96 | 97 | ### Heroku 98 | 99 | ```bash 100 | $ heroku create 101 | $ heroku config:add CARBON_URLS=... 102 | $ heroku config:add PREFIXES=... 103 | $ heroku config:add BACKSTOP_AUTH=... (optional) 104 | $ git push heroku master 105 | ``` 106 | 107 | ## License 108 | 109 | Backstop is distributed under a 3-clause BSD license. 110 | 111 | ## Thanks 112 | 113 | Thanks to Michael Gorsuch (@gorsuch) for his work on the collectd parser and the "Mitt" application that preceded Backstop. 114 | 115 | -------------------------------------------------------------------------------- /lib/backstop/web.rb: -------------------------------------------------------------------------------- 1 | require 'sinatra' 2 | require 'json' 3 | require 'time' 4 | 5 | require 'backstop' 6 | 7 | module Backstop 8 | class Application < Sinatra::Base 9 | configure do 10 | enable :logging 11 | require 'newrelic_rpm' 12 | @@publisher = nil 13 | end 14 | 15 | before do 16 | protected! unless request.path == '/health' 17 | end 18 | 19 | helpers do 20 | def protected! 21 | return unless ENV['BACKSTOP_AUTH'] 22 | return if authorized? 23 | headers['WWW-Authenticate'] = 'Basic realm="Restricted Area"' 24 | halt 401, "Not authorized\n" 25 | end 26 | def authorized? 27 | @auth ||= Rack::Auth::Basic::Request.new(request.env) 28 | @auth.provided? and @auth.basic? and @auth.credentials and @auth.credentials == ENV['BACKSTOP_AUTH'].split(':') 29 | end 30 | def publisher 31 | @@publisher ||= Backstop::Publisher.new(Config.carbon_urls, :api_key => Config.api_key) 32 | end 33 | def send(metric, value, time) 34 | begin 35 | publisher.publish(metric, value, time) 36 | rescue 37 | publisher.close_all 38 | @@publisher = nil 39 | end 40 | end 41 | end 42 | 43 | get '/health' do 44 | {'health' => 'ok'}.to_json 45 | end 46 | 47 | post '/collectd' do 48 | begin 49 | data = JSON.parse(request.body.read) 50 | rescue JSON::ParserError 51 | halt 400, 'JSON is required' 52 | end 53 | data.each do |item| 54 | results = CollectdData.new(item).parse 55 | results.each do |r| 56 | r['source'] = 'collectd' 57 | halt 400, 'missing fields' unless (r[:cloud] && r[:slot] && r[:id] && r[:metric] && r[:value] && r[:measure_time]) 58 | r[:cloud].gsub!(/\./, '-') 59 | send("mitt.#{r[:cloud]}.#{r[:slot]}.#{r[:id]}.#{r[:metric]}", r[:value], r[:measure_time]) 60 | end 61 | end 62 | 'ok' 63 | end 64 | 65 | post '/github' do 66 | begin 67 | data = JSON.parse(params[:payload]) 68 | rescue JSON::ParserError 69 | halt 400, 'JSON is required' 70 | end 71 | halt 400, 'missing fields' unless (data['repository'] && data['commits']) 72 | data['source'] = 'github' 73 | data['ref'].gsub!(/\//, '.') 74 | data['commits'].each do |commit| 75 | repo = data['repository']['name'] 76 | author = commit['author']['email'].gsub(/[\.@]/, '-') 77 | measure_time = DateTime.parse(commit['timestamp']).strftime('%s') 78 | send("#{data['source']}.#{repo}.#{data['ref']}.#{author}.#{commit['id']}", 1, measure_time) 79 | end 80 | 'ok' 81 | end 82 | 83 | post '/pagerduty' do 84 | begin 85 | incident = params 86 | rescue 87 | halt 400, 'unknown payload' 88 | end 89 | case incident['service']['name'] 90 | when 'Pingdom' 91 | metric = "pingdom.#{incident['incident_key'].gsub(/\./, '_').gsub(/[\(\)]/, '').gsub(/\s+/, '.')}" 92 | when 'nagios' 93 | data = incident['trigger_summary_data'] 94 | outage = data['SERVICEDESC'] === '' ? 'host_down' : data['SERVICEDESC'] 95 | begin 96 | metric = "nagios.#{data['HOSTNAME'].gsub(/\./, '_')}.#{outage}" 97 | rescue 98 | puts "UNKNOWN ALERT: #{incident.to_json}" 99 | halt 400, 'unknown alert' 100 | end 101 | when 'Enterprise Zendesk' 102 | metric = "enterprise.zendesk.#{incident['service']['id']}" 103 | else 104 | puts "UNKNOWN ALERT: #{incident.to_json}" 105 | halt 400, 'unknown alert' 106 | end 107 | send("alerts.#{metric}", 1, Time.parse(incident['created_on']).to_i) 108 | 'ok' 109 | end 110 | 111 | post '/publish/:name' do 112 | begin 113 | data = JSON.parse(request.body.read) 114 | rescue JSON::ParserError 115 | halt 400, 'JSON is required' 116 | end 117 | if Config.prefixes.include?(params[:name]) 118 | if data.kind_of? Array 119 | data.each do |item| 120 | item['source'] = params[:name] 121 | halt 400, 'missing fields' unless (item['metric'] && item['value'] && item['measure_time']) 122 | send("#{item['source']}.#{item['metric']}", item['value'], item['measure_time']) 123 | end 124 | else 125 | data['source'] = params[:name] 126 | halt 400, 'missing fields' unless (data['metric'] && data['value'] && data['measure_time']) 127 | send("#{data['source']}.#{data['metric']}", data['value'], data['measure_time']) 128 | end 129 | 'ok' 130 | else 131 | halt 404, 'unknown prefix' 132 | end 133 | end 134 | end 135 | end 136 | --------------------------------------------------------------------------------