├── lib ├── nsa │ ├── version.rb │ ├── statsd.rb │ ├── collectors │ │ ├── null.rb │ │ ├── action_controller.rb │ │ ├── active_support_cache.rb │ │ ├── active_record.rb │ │ └── sidekiq.rb │ └── statsd │ │ ├── informant.rb │ │ ├── subscriber.rb │ │ ├── async_publisher.rb │ │ └── publisher.rb └── nsa.rb ├── .travis.yml ├── Gemfile ├── bin ├── setup └── console ├── .gitignore ├── Rakefile ├── test ├── lib │ └── nsa │ │ ├── statsd │ │ ├── informant_test.rb │ │ ├── subscriber_test.rb │ │ └── publisher_test.rb │ │ └── collectors │ │ ├── active_record_test.rb │ │ ├── action_controller_test.rb │ │ └── active_support_cache_test.rb └── test_helper.rb ├── nsa.gemspec └── README.md /lib/nsa/version.rb: -------------------------------------------------------------------------------- 1 | module NSA 2 | VERSION = "0.2.8" 3 | end 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.6.5 4 | before_install: gem install bundler -v 2.1.4 5 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in nsa.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | 5 | bundle install 6 | 7 | # Do any other automated setup that you need to do here 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | /tags 11 | *.gem -------------------------------------------------------------------------------- /lib/nsa/statsd.rb: -------------------------------------------------------------------------------- 1 | require "nsa/statsd/subscriber" 2 | require "nsa/statsd/publisher" 3 | require "nsa/statsd/async_publisher" 4 | require "nsa/statsd/informant" 5 | -------------------------------------------------------------------------------- /lib/nsa/collectors/null.rb: -------------------------------------------------------------------------------- 1 | module NSA 2 | module Collectors 3 | module Null 4 | 5 | def self.collect(*_) 6 | end 7 | 8 | end 9 | end 10 | end 11 | 12 | -------------------------------------------------------------------------------- /lib/nsa.rb: -------------------------------------------------------------------------------- 1 | module NSA 2 | 3 | def self.inform_statsd(backend) 4 | yield ::NSA::Statsd::Informant 5 | ::NSA::Statsd::Informant.listen(backend) 6 | end 7 | 8 | end 9 | 10 | require "nsa/version" 11 | require "nsa/statsd" 12 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rake/testtask" 3 | 4 | Rake::TestTask.new(:test) do |t| 5 | t.libs << "test" 6 | t.libs << "lib" 7 | t.test_files = FileList['test/**/*_test.rb'] 8 | end 9 | 10 | task :default => :test 11 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "nsa" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start 15 | -------------------------------------------------------------------------------- /test/lib/nsa/statsd/informant_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "../../../test_helper" 2 | 3 | class ::NSA::Statsd::InformantTest < ::Minitest::Test 4 | 5 | def test_collect_action_controller 6 | collector = mock 7 | collector.expects(:respond_to?).with(:collect).returns(true) 8 | collector.expects(:collect).with("foo") 9 | ::NSA::Statsd::Informant.collect(collector, "foo") 10 | end 11 | 12 | def test_null_collector 13 | collector = ::NSA::Statsd::Informant::COLLECTOR_TYPES[:skibidy] 14 | assert_equal(::NSA::Collectors::Null, collector, "not a null collector") 15 | end 16 | 17 | end 18 | 19 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__)) 2 | require "nsa" 3 | 4 | require "byebug" 5 | require "minitest/pride" 6 | require "minitest/unit" 7 | require "mocha/minitest" 8 | 9 | require "minitest/autorun" 10 | 11 | module Mocha::ParameterMatchers 12 | class InDelta < Mocha::ParameterMatchers::Base 13 | def initialize(value, delta=0.001) 14 | @value, @delta = value, delta 15 | @range = (-@delta...@delta) 16 | end 17 | 18 | def matches?(available_parameters) 19 | parameter = available_parameters.shift 20 | @range.cover?(parameter - @value) 21 | end 22 | 23 | def mocha_inspect 24 | "in_delta(#{@value.mocha_inspect}, #{@delta.mocha_inspect})" 25 | end 26 | end 27 | 28 | def in_delta(*args) 29 | InDelta.new(*args) 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/nsa/statsd/informant.rb: -------------------------------------------------------------------------------- 1 | require "nsa/collectors/action_controller" 2 | require "nsa/collectors/active_record" 3 | require "nsa/collectors/active_support_cache" 4 | require "nsa/collectors/null" 5 | require "nsa/collectors/sidekiq" 6 | 7 | module NSA 8 | module Statsd 9 | module Informant 10 | extend ::NSA::Statsd::Subscriber 11 | 12 | COLLECTOR_TYPES = ::Hash.new(::NSA::Collectors::Null).merge({ 13 | :action_controller => ::NSA::Collectors::ActionController, 14 | :active_record => ::NSA::Collectors::ActiveRecord, 15 | :active_support_cache => ::NSA::Collectors::ActiveSupportCache, 16 | :sidekiq => ::NSA::Collectors::Sidekiq 17 | }).freeze 18 | 19 | def self.collect(collector, key_prefix) 20 | collector = COLLECTOR_TYPES[collector.to_sym] unless collector.respond_to?(:collect) 21 | collector.collect(key_prefix) 22 | end 23 | 24 | def self.listen(backend) 25 | statsd_subscribe(backend) 26 | end 27 | 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/nsa/collectors/action_controller.rb: -------------------------------------------------------------------------------- 1 | require "active_support/notifications" 2 | require "nsa/statsd/publisher" 3 | 4 | module NSA 5 | module Collectors 6 | module ActionController 7 | extend ::NSA::Statsd::Publisher 8 | 9 | def self.collect(key_prefix) 10 | ::ActiveSupport::Notifications.subscribe(/process_action.action_controller/) do |*args| 11 | event = ::ActiveSupport::Notifications::Event.new(*args) 12 | controller = event.payload[:controller] 13 | action = event.payload[:action] 14 | format = event.payload[:format] || "all" 15 | format = "all" if format == "*/*" 16 | status = event.payload[:status] 17 | key = "#{key_prefix}.#{controller}.#{action}.#{format}" 18 | 19 | statsd_timing("#{key}.total_duration", event.duration) 20 | statsd_timing("#{key}.db_time", event.payload[:db_runtime]) 21 | statsd_timing("#{key}.view_time", event.payload[:view_runtime]) 22 | statsd_increment("#{key}.status.#{status}") 23 | end 24 | end 25 | 26 | end 27 | end 28 | end 29 | 30 | -------------------------------------------------------------------------------- /lib/nsa/collectors/active_support_cache.rb: -------------------------------------------------------------------------------- 1 | require "active_support/notifications" 2 | require "nsa/statsd/publisher" 3 | 4 | module NSA 5 | module Collectors 6 | module ActiveSupportCache 7 | extend ::NSA::Statsd::Publisher 8 | 9 | CACHE_TYPES = { 10 | "cache_delete.active_support" => :delete, 11 | "cache_exist?.active_support" => :exist?, 12 | "cache_fetch_hit.active_support" => :fetch_hit, 13 | "cache_generate.active_support" => :generate, 14 | "cache_read.active_support" => :read, 15 | "cache_write.active_support" => :write, 16 | }.freeze 17 | 18 | def self.collect(key_prefix) 19 | ::ActiveSupport::Notifications.subscribe(/cache_[^.]+.active_support/) do |*event_args| 20 | event = ::ActiveSupport::Notifications::Event.new(*event_args) 21 | cache_type = CACHE_TYPES.fetch(event.name) do 22 | event.name.split(".").first.gsub(/^cache_/, "") 23 | end 24 | 25 | if cache_type == :read 26 | cache_type = event.payload[:hit] ? :read_hit : :read_miss 27 | end 28 | 29 | stat_name = "#{key_prefix}.#{cache_type}.duration" 30 | statsd_timing(stat_name, event.duration) 31 | end 32 | end 33 | 34 | end 35 | end 36 | end 37 | 38 | -------------------------------------------------------------------------------- /lib/nsa/collectors/active_record.rb: -------------------------------------------------------------------------------- 1 | require "nsa/statsd/publisher" 2 | 3 | module NSA 4 | module Collectors 5 | module ActiveRecord 6 | extend ::NSA::Statsd::Publisher 7 | 8 | # Ordered by most common query type 9 | MATCHERS = [ 10 | [ :select, /^\s*SELECT.+?FROM\s+"?([^".\s),]+)"?/im ], 11 | [ :insert, /^\s*INSERT INTO\s+"?([^".\s]+)"?/im ], 12 | [ :update, /^\s*UPDATE\s+"?([^".\s]+)"?/im ], 13 | [ :delete, /^\s*DELETE.+FROM\s+"?([^".\s]+)"?/im ] 14 | ].freeze 15 | 16 | EMPTY_MATCH_RESULT = [] 17 | 18 | def self.collect(key_prefix) 19 | ::ActiveSupport::Notifications.subscribe("sql.active_record") do |_, start, finish, _id, payload| 20 | query_type, table_name = match_query(payload[:sql]) 21 | unless query_type.nil? 22 | stat_name = "#{key_prefix}.tables.#{table_name}.queries.#{query_type}.duration" 23 | duration_ms = (finish - start) * 1000 24 | statsd_timing(stat_name, duration_ms) 25 | end 26 | end 27 | end 28 | 29 | def self.match_query(sql) 30 | MATCHERS 31 | .lazy 32 | .map { |(type, regex)| 33 | match = (sql.match(regex) || EMPTY_MATCH_RESULT) 34 | [ type, match[1] ] 35 | } 36 | .detect { |(_, table_name)| ! table_name.nil? } 37 | end 38 | 39 | end 40 | end 41 | end 42 | 43 | -------------------------------------------------------------------------------- /nsa.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path("../lib", __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require "nsa/version" 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "nsa" 8 | spec.version = ::NSA::VERSION 9 | spec.authors = ["BJ Neilsen"] 10 | spec.email = ["bj.neilsen@gmail.com"] 11 | 12 | spec.summary = %q{Publish Rails application metrics to statsd} 13 | spec.description = %q{Listen to your Rails ActiveSupport::Notifications and deliver to a Statsd compatible backend} 14 | spec.homepage = "https://github.com/localshred/nsa" 15 | spec.licenses = ["MIT"] 16 | 17 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 18 | spec.bindir = "exe" 19 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 20 | spec.require_paths = ["lib"] 21 | 22 | spec.add_dependency "activesupport", "< 7.2", ">= 4.2" 23 | spec.add_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.2" 24 | spec.add_dependency "sidekiq", ">= 3.5" 25 | spec.add_dependency "statsd-ruby", "~> 1.4", ">= 1.4.0" 26 | 27 | spec.add_development_dependency "bundler", "~> 2.1" 28 | spec.add_development_dependency "rake", "~> 13.0" 29 | spec.add_development_dependency "minitest", "~> 5.0" 30 | spec.add_development_dependency "mocha", "~> 1.11" 31 | spec.add_development_dependency "byebug", "~> 10" 32 | end 33 | -------------------------------------------------------------------------------- /lib/nsa/statsd/subscriber.rb: -------------------------------------------------------------------------------- 1 | require "statsd-ruby" 2 | 3 | module NSA 4 | module Statsd 5 | module Subscriber 6 | 7 | EXPECTED_RESPONDABLE_METHODS = %i( count decrement gauge increment set time timing ).freeze 8 | 9 | def statsd_subscribe(backend) 10 | unless backend_valid?(backend) 11 | fail "Backend must respond to the following methods:\n\t#{EXPECTED_RESPONDABLE_METHODS.join(", ")}" 12 | end 13 | 14 | ::ActiveSupport::Notifications.subscribe(/.statsd$/) do |name, start, finish, id, payload| 15 | __send_event_to_statsd(backend, name, start, finish, id, payload) 16 | end 17 | end 18 | 19 | def __send_event_to_statsd(backend, name, start, finish, id, payload) 20 | action = name.to_s.split(".").first || :count 21 | 22 | key_name = payload[:key] 23 | sample_rate = payload.fetch(:sample_rate, 1) 24 | 25 | case action.to_sym 26 | when :count, :timing, :set, :gauge then 27 | value = payload.fetch(:value) { 1 } 28 | backend.__send__(action, key_name, value, sample_rate) 29 | when :increment, :decrement then 30 | backend.__send__(action, key_name, sample_rate) 31 | end 32 | end 33 | 34 | def backend_valid?(backend) 35 | EXPECTED_RESPONDABLE_METHODS.all? do |method_name| 36 | backend.respond_to?(method_name) 37 | end 38 | end 39 | 40 | end 41 | end 42 | end 43 | 44 | -------------------------------------------------------------------------------- /lib/nsa/statsd/async_publisher.rb: -------------------------------------------------------------------------------- 1 | require "concurrent" 2 | 3 | module NSA 4 | module Statsd 5 | module AsyncPublisher 6 | include ::NSA::Statsd::Publisher 7 | 8 | def async_statsd_count(key, sample_rate = 1, &block) 9 | return unless sample_rate == 1 || rand < sample_rate 10 | 11 | ::Concurrent::Promise.execute(&block).then do |value| 12 | statsd_count(key, value) 13 | end 14 | end 15 | 16 | def async_statsd_gauge(key, sample_rate = 1, &block) 17 | return unless sample_rate == 1 || rand < sample_rate 18 | 19 | ::Concurrent::Promise.execute(&block).then do |value| 20 | statsd_gauge(key, value) 21 | end 22 | end 23 | 24 | def async_statsd_set(key, sample_rate = 1, &block) 25 | return unless sample_rate == 1 || rand < sample_rate 26 | 27 | ::Concurrent::Promise.execute(&block).then do |value| 28 | statsd_set(key, value) 29 | end 30 | end 31 | 32 | def async_statsd_time(key, sample_rate = 1, &block) 33 | return unless sample_rate == 1 || rand < sample_rate 34 | 35 | ::Concurrent::Future.execute do 36 | statsd_time(key, &block) 37 | end 38 | end 39 | 40 | def async_statsd_timing(key, sample_rate = 1, &block) 41 | return unless sample_rate == 1 || rand < sample_rate 42 | 43 | ::Concurrent::Promise.execute(&block).then do |value| 44 | statsd_timing(key, value) 45 | end 46 | end 47 | 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/nsa/statsd/publisher.rb: -------------------------------------------------------------------------------- 1 | module NSA 2 | module Statsd 3 | module Publisher 4 | 5 | def statsd_count(key, value = 1, sample_rate = nil) 6 | __statsd_publish(:count, key, value, sample_rate) 7 | end 8 | 9 | def statsd_decrement(key, sample_rate = nil) 10 | __statsd_publish(:decrement, key, 1, sample_rate) 11 | end 12 | 13 | def statsd_gauge(key, value = 1, sample_rate = nil) 14 | __statsd_publish(:gauge, key, value, sample_rate) 15 | end 16 | 17 | def statsd_increment(key, sample_rate = nil) 18 | __statsd_publish(:increment, key, 1, sample_rate) 19 | end 20 | 21 | def statsd_set(key, value = 1, sample_rate = nil) 22 | __statsd_publish(:set, key, value, sample_rate) 23 | end 24 | 25 | def statsd_time(key, sample_rate = nil, &block) 26 | start = Time.now 27 | result = block.call unless block.nil? 28 | statsd_timing(key, ((Time.now - start) * 1000).round, sample_rate) 29 | result 30 | end 31 | 32 | def statsd_timing(key, value = 1, sample_rate = nil) 33 | __statsd_publish(:timing, key, value, sample_rate) 34 | end 35 | 36 | def __statsd_publish(stat_type, key, value = nil, sample_rate = nil) 37 | payload = { :key => key } 38 | payload.merge!({ :value => value }) if value 39 | payload.merge!({ :sample_rate => sample_rate }) if sample_rate 40 | 41 | ::ActiveSupport::Notifications.instrument("#{stat_type}.statsd", payload) 42 | end 43 | 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /test/lib/nsa/statsd/subscriber_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "../../../test_helper" 2 | 3 | class SubscriberTest < ::Minitest::Test 4 | 5 | InformantTest = Class.new do 6 | include ::NSA::Statsd::Subscriber 7 | end 8 | 9 | attr_accessor :informant, :backend 10 | 11 | NAME = "count.statsd" 12 | START = ::Time.now - 3 13 | FINISH = ::Time.now - 2 14 | NOTIFICATION_ID = ::SecureRandom.hex 15 | KEY = :total_users 16 | VALUE = 321 17 | SAMPLE_RATE = 0.8 18 | PAYLOAD = { :key => KEY, :value => VALUE, :sample_rate => SAMPLE_RATE } 19 | 20 | def setup 21 | @informant = InformantTest.new 22 | @backend = ::Statsd.new 23 | end 24 | 25 | def test_statsd_subscribe 26 | informant.expects(:__send_event_to_statsd).with(backend, NAME, START, FINISH, NOTIFICATION_ID, PAYLOAD) 27 | ::ActiveSupport::Notifications.expects(:subscribe).with(/.statsd$/).yields(NAME, START, FINISH, NOTIFICATION_ID, PAYLOAD) 28 | informant.statsd_subscribe(backend) 29 | end 30 | 31 | def test_key_value_event_types 32 | backend.expects(:count).with(KEY, VALUE, SAMPLE_RATE) 33 | backend.expects(:gauge).with(KEY, VALUE, SAMPLE_RATE) 34 | backend.expects(:set).with(KEY, VALUE, SAMPLE_RATE) 35 | backend.expects(:timing).with(KEY, VALUE, SAMPLE_RATE) 36 | informant.__send_event_to_statsd(backend, "count.statsd", START, FINISH, NOTIFICATION_ID, PAYLOAD) 37 | informant.__send_event_to_statsd(backend, "gauge.statsd", START, FINISH, NOTIFICATION_ID, PAYLOAD) 38 | informant.__send_event_to_statsd(backend, "set.statsd", START, FINISH, NOTIFICATION_ID, PAYLOAD) 39 | informant.__send_event_to_statsd(backend, "timing.statsd", START, FINISH, NOTIFICATION_ID, PAYLOAD) 40 | end 41 | 42 | def test_key_event_types 43 | backend.expects(:decrement).with(KEY, SAMPLE_RATE) 44 | backend.expects(:increment).with(KEY, SAMPLE_RATE) 45 | informant.__send_event_to_statsd(backend, "decrement.statsd", START, FINISH, NOTIFICATION_ID, PAYLOAD) 46 | informant.__send_event_to_statsd(backend, "increment.statsd", START, FINISH, NOTIFICATION_ID, PAYLOAD) 47 | end 48 | 49 | 50 | end 51 | -------------------------------------------------------------------------------- /lib/nsa/collectors/sidekiq.rb: -------------------------------------------------------------------------------- 1 | require "nsa/statsd/publisher" 2 | 3 | module NSA 4 | module Collectors 5 | class Sidekiq 6 | include ::NSA::Statsd::Publisher 7 | 8 | def self.collect(key_prefix) 9 | require "sidekiq" 10 | 11 | ::Sidekiq.configure_server do |config| 12 | config.server_middleware do |chain| 13 | chain.add(::NSA::Collectors::Sidekiq, key_prefix) 14 | end 15 | end 16 | rescue ::LoadError => exception 17 | $stderr.puts("[LoadError] Failed to load sidekiq!", exception.message, *(exception.backtrace)) 18 | end 19 | 20 | attr_accessor :key_prefix 21 | private :key_prefix= 22 | 23 | def initialize(key_prefix) 24 | self.key_prefix = key_prefix.to_s.split(".") 25 | end 26 | 27 | def call(worker, message, queue_name) 28 | worker_name = worker.class.name.gsub("::", ".") 29 | 30 | statsd_time(make_key(worker_name, :processing_time)) do 31 | yield 32 | end 33 | 34 | statsd_increment(make_key(worker_name, :success)) 35 | rescue => exception 36 | statsd_increment(make_key(worker_name, :failure)) 37 | fail exception 38 | ensure 39 | publish_overall_stats 40 | publish_queue_size_and_latency(queue_name) 41 | end 42 | 43 | private 44 | 45 | def publish_overall_stats 46 | stats = ::Sidekiq::Stats.new 47 | statsd_gauge(make_key(:dead_size), stats.dead_size) 48 | statsd_gauge(make_key(:enqueued), stats.enqueued) 49 | statsd_gauge(make_key(:failed), stats.failed) 50 | statsd_gauge(make_key(:processed), stats.processed) 51 | statsd_gauge(make_key(:processes_size), stats.processes_size) 52 | statsd_gauge(make_key(:retry_size), stats.retry_size) 53 | statsd_gauge(make_key(:scheduled_size), stats.scheduled_size) 54 | statsd_gauge(make_key(:workers_size), stats.workers_size) 55 | end 56 | 57 | def publish_queue_size_and_latency(queue_name) 58 | queue = ::Sidekiq::Queue.new(queue_name) 59 | statsd_gauge(make_key(:queues, queue_name, :enqueued), queue.size) 60 | 61 | if queue.respond_to?(:latency) 62 | statsd_gauge(make_key(:queues, queue_name, :latency), queue.latency) 63 | end 64 | end 65 | 66 | def make_key(*args) 67 | (key_prefix + args).compact.join(".") 68 | end 69 | 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /test/lib/nsa/collectors/active_record_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "../../../test_helper" 2 | 3 | class ActiveRecordTest < ::Minitest::Test 4 | 5 | def test_collect_delete_query 6 | duration = 0.1 7 | event = { :sql => %q{DELETE FROM "users" WHERE "users".id = 1} } 8 | expect_subscriber(event, duration) 9 | 10 | ::NSA::Collectors::ActiveRecord.expects(:statsd_timing).with("db.tables.users.queries.delete.duration", in_delta(duration * 1000)) 11 | ::NSA::Collectors::ActiveRecord.collect("db") 12 | end 13 | 14 | def test_collect_insert_query 15 | duration = 0.2 16 | event = { :sql => %q{INSERT INTO "users" ("id", "name") VALUES (1, 'Bob')} } 17 | expect_subscriber(event, duration) 18 | 19 | ::NSA::Collectors::ActiveRecord.expects(:statsd_timing).with("db.tables.users.queries.insert.duration", in_delta(duration * 1000)) 20 | ::NSA::Collectors::ActiveRecord.collect("db") 21 | end 22 | 23 | def test_collect_select_query 24 | duration = 0.3 25 | event = { :sql => %q{SELECT "users".id, "users".name FROM "users"} } 26 | expect_subscriber(event, duration) 27 | 28 | ::NSA::Collectors::ActiveRecord.expects(:statsd_timing).with("db.tables.users.queries.select.duration", in_delta(duration * 1000)) 29 | ::NSA::Collectors::ActiveRecord.collect("db") 30 | end 31 | 32 | def test_collect_select_query_multiline 33 | duration = 0.3 34 | query = %q{ 35 | SELECT 36 | "users".id, 37 | "users".name 38 | FROM "users" 39 | INNER JOIN ( 40 | SELECT * FROM accounts 41 | ) 42 | } 43 | event = { :sql => query } 44 | expect_subscriber(event, duration) 45 | 46 | ::NSA::Collectors::ActiveRecord.expects(:statsd_timing).with("db.tables.users.queries.select.duration", in_delta(duration * 1000)) 47 | ::NSA::Collectors::ActiveRecord.collect("db") 48 | end 49 | 50 | def test_collect_update_query 51 | duration = 0.4 52 | event = { :sql => %q{UPDATE "users" SET "users".name = 'Joe' WHERE "users".id = 1} } 53 | expect_subscriber(event, duration) 54 | 55 | ::NSA::Collectors::ActiveRecord.expects(:statsd_timing).with("db.tables.users.queries.update.duration", in_delta(duration * 1000)) 56 | ::NSA::Collectors::ActiveRecord.collect("db") 57 | end 58 | 59 | private 60 | 61 | def expect_subscriber(payload, duration = 1) 62 | finish = ::Time.now 63 | start = finish - duration 64 | 65 | ::ActiveSupport::Notifications 66 | .expects(:subscribe) 67 | .with("sql.active_record") 68 | .yields("sql.active_record", start, finish, ::SecureRandom.hex, payload) 69 | end 70 | 71 | end 72 | -------------------------------------------------------------------------------- /test/lib/nsa/collectors/action_controller_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "../../../test_helper" 2 | 3 | class ActionControllerTest < ::Minitest::Test 4 | 5 | def test_collect 6 | duration = 0.7 7 | event = { :controller => "UsersController", 8 | :action => :index, 9 | :format => :html, 10 | :status => 200, 11 | :db_runtime => 200, 12 | :view_runtime => 100 } 13 | expect_subscriber(event, duration) 14 | 15 | ::NSA::Collectors::ActionController.expects(:statsd_timing).with("web.UsersController.index.html.total_duration", in_delta(duration * 1000)) 16 | ::NSA::Collectors::ActionController.expects(:statsd_timing).with("web.UsersController.index.html.db_time", event[:db_runtime]) 17 | ::NSA::Collectors::ActionController.expects(:statsd_timing).with("web.UsersController.index.html.view_time", event[:view_runtime]) 18 | ::NSA::Collectors::ActionController.expects(:statsd_increment).with("web.UsersController.index.html.status.#{event[:status]}") 19 | ::NSA::Collectors::ActionController.collect("web") 20 | end 21 | 22 | def test_collect_nil_format 23 | duration = 0.7 24 | event = { :controller => "UsersController", 25 | :action => :index, 26 | :format => nil, 27 | :status => 200, 28 | :db_runtime => 200, 29 | :view_runtime => 100 } 30 | expect_subscriber(event, duration) 31 | 32 | ::NSA::Collectors::ActionController.expects(:statsd_timing).with("web.UsersController.index.all.total_duration", in_delta(duration * 1000)) 33 | ::NSA::Collectors::ActionController.expects(:statsd_timing).with("web.UsersController.index.all.db_time", event[:db_runtime]) 34 | ::NSA::Collectors::ActionController.expects(:statsd_timing).with("web.UsersController.index.all.view_time", event[:view_runtime]) 35 | ::NSA::Collectors::ActionController.expects(:statsd_increment).with("web.UsersController.index.all.status.#{event[:status]}") 36 | ::NSA::Collectors::ActionController.collect("web") 37 | end 38 | 39 | def test_collect_splat_format 40 | duration = 0.7 41 | event = { :controller => "UsersController", 42 | :action => :index, 43 | :format => "*/*", 44 | :status => 200, 45 | :db_runtime => 200, 46 | :view_runtime => 100 } 47 | expect_subscriber(event, duration) 48 | 49 | ::NSA::Collectors::ActionController.expects(:statsd_timing).with("web.UsersController.index.all.total_duration", in_delta(duration * 1000)) 50 | ::NSA::Collectors::ActionController.expects(:statsd_timing).with("web.UsersController.index.all.db_time", event[:db_runtime]) 51 | ::NSA::Collectors::ActionController.expects(:statsd_timing).with("web.UsersController.index.all.view_time", event[:view_runtime]) 52 | ::NSA::Collectors::ActionController.expects(:statsd_increment).with("web.UsersController.index.all.status.#{event[:status]}") 53 | ::NSA::Collectors::ActionController.collect("web") 54 | end 55 | 56 | private 57 | 58 | def expect_subscriber(payload, duration = 1) 59 | finish = ::Time.now 60 | start = finish - duration 61 | 62 | ::ActiveSupport::Notifications 63 | .expects(:subscribe) 64 | .with(/process_action.action_controller/) 65 | .yields("process_action.action_controller", start, finish, ::SecureRandom.hex, payload) 66 | end 67 | 68 | end 69 | -------------------------------------------------------------------------------- /test/lib/nsa/collectors/active_support_cache_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "../../../test_helper" 2 | 3 | class ActiveSupportCacheTest < ::Minitest::Test 4 | def test_collect_cache_delete 5 | duration = 0.1 6 | event = {} 7 | expect_subscriber("cache_delete.active_support", event, duration) 8 | 9 | ::NSA::Collectors::ActiveSupportCache.expects(:statsd_timing).with("cache.delete.duration", in_delta(duration * 1000)) 10 | ::NSA::Collectors::ActiveSupportCache.collect("cache") 11 | end 12 | 13 | def test_collect_cache_exist? 14 | duration = 0.1 15 | event = {} 16 | expect_subscriber("cache_exist?.active_support", event, duration) 17 | 18 | ::NSA::Collectors::ActiveSupportCache.expects(:statsd_timing).with("cache.exist?.duration", in_delta(duration * 1000)) 19 | ::NSA::Collectors::ActiveSupportCache.collect("cache") 20 | end 21 | 22 | def test_collect_cache_fetch_hit 23 | duration = 0.1 24 | event = {} 25 | expect_subscriber("cache_fetch_hit.active_support", event, duration) 26 | 27 | ::NSA::Collectors::ActiveSupportCache.expects(:statsd_timing).with("cache.fetch_hit.duration", in_delta(duration * 1000)) 28 | ::NSA::Collectors::ActiveSupportCache.collect("cache") 29 | end 30 | 31 | def test_collect_cache_generate 32 | duration = 0.1 33 | event = {} 34 | expect_subscriber("cache_generate.active_support", event, duration) 35 | 36 | ::NSA::Collectors::ActiveSupportCache.expects(:statsd_timing).with("cache.generate.duration", in_delta(duration * 1000)) 37 | ::NSA::Collectors::ActiveSupportCache.collect("cache") 38 | end 39 | 40 | def test_collect_cache_read_hit 41 | duration = 0.1 42 | event = { :hit => true } 43 | expect_subscriber("cache_read.active_support", event, duration) 44 | 45 | ::NSA::Collectors::ActiveSupportCache.expects(:statsd_timing).with("cache.read_hit.duration", in_delta(duration * 1000)) 46 | ::NSA::Collectors::ActiveSupportCache.collect("cache") 47 | end 48 | 49 | def test_collect_cache_read_miss 50 | duration = 0.1 51 | event = { :hit => false } 52 | expect_subscriber("cache_read.active_support", event, duration) 53 | 54 | ::NSA::Collectors::ActiveSupportCache.expects(:statsd_timing).with("cache.read_miss.duration", in_delta(duration * 1000)) 55 | ::NSA::Collectors::ActiveSupportCache.collect("cache") 56 | end 57 | 58 | def test_collect_cache_write 59 | duration = 0.1 60 | event = {} 61 | expect_subscriber("cache_write.active_support", event, duration) 62 | 63 | ::NSA::Collectors::ActiveSupportCache.expects(:statsd_timing).with("cache.write.duration", in_delta(duration * 1000)) 64 | ::NSA::Collectors::ActiveSupportCache.collect("cache") 65 | end 66 | 67 | def test_collect_cache_some_other_key 68 | duration = 0.1 69 | event = {} 70 | expect_subscriber("cache_some_other_key.active_support", event, duration) 71 | 72 | ::NSA::Collectors::ActiveSupportCache.expects(:statsd_timing).with("cache.some_other_key.duration", in_delta(duration * 1000)) 73 | ::NSA::Collectors::ActiveSupportCache.collect("cache") 74 | end 75 | 76 | private 77 | 78 | def expect_subscriber(event_name, payload, duration = 1) 79 | finish = ::Time.now 80 | start = finish - duration 81 | 82 | ::ActiveSupport::Notifications 83 | .expects(:subscribe) 84 | .with(/cache_[^.]+.active_support/) 85 | .yields(event_name, start, finish, ::SecureRandom.hex, payload) 86 | end 87 | 88 | end 89 | -------------------------------------------------------------------------------- /test/lib/nsa/statsd/publisher_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "../../../test_helper" 2 | 3 | class PublisherTest < ::Minitest::Test 4 | 5 | CollectorTest = Class.new do 6 | include ::NSA::Statsd::Publisher 7 | end 8 | 9 | attr_accessor :collector 10 | 11 | def setup 12 | @collector = CollectorTest.new 13 | end 14 | 15 | def test_count 16 | collector.expects(:__statsd_publish).with(:count, :foo, 5, 0.5) 17 | collector.expects(:__statsd_publish).with(:count, :bar, 15, nil) 18 | collector.expects(:__statsd_publish).with(:count, :baz, 1, nil) 19 | collector.statsd_count(:foo, 5, 0.5) 20 | collector.statsd_count(:bar, 15) 21 | collector.statsd_count(:baz) 22 | end 23 | 24 | def test_decrement 25 | collector.expects(:__statsd_publish).with(:decrement, :foo, 1, 0.5) 26 | collector.expects(:__statsd_publish).with(:decrement, :bar, 1, nil) 27 | collector.statsd_decrement(:foo, 0.5) 28 | collector.statsd_decrement(:bar) 29 | end 30 | 31 | def test_gauge 32 | collector.expects(:__statsd_publish).with(:gauge, :foo, 5, 0.5) 33 | collector.expects(:__statsd_publish).with(:gauge, :bar, 15, nil) 34 | collector.expects(:__statsd_publish).with(:gauge, :baz, 1, nil) 35 | collector.statsd_gauge(:foo, 5, 0.5) 36 | collector.statsd_gauge(:bar, 15) 37 | collector.statsd_gauge(:baz) 38 | end 39 | 40 | def test_increment 41 | collector.expects(:__statsd_publish).with(:increment, :foo, 1, 0.5) 42 | collector.expects(:__statsd_publish).with(:increment, :bar, 1, nil) 43 | collector.statsd_increment(:foo, 0.5) 44 | collector.statsd_increment(:bar) 45 | end 46 | 47 | def test_set 48 | collector.expects(:__statsd_publish).with(:set, :foo, 5, 0.5) 49 | collector.expects(:__statsd_publish).with(:set, :bar, 15, nil) 50 | collector.expects(:__statsd_publish).with(:set, :baz, 1, nil) 51 | collector.statsd_set(:foo, 5, 0.5) 52 | collector.statsd_set(:bar, 15) 53 | collector.statsd_set(:baz) 54 | end 55 | 56 | def test_time 57 | collector.expects(:__statsd_publish).with(:timing, :foo, kind_of(Integer), 0.5) 58 | collector.expects(:__statsd_publish).with(:timing, :bar, kind_of(Integer), nil) 59 | collector.statsd_time(:foo, 0.5) { 1 } 60 | collector.statsd_time(:bar) { 1 } 61 | end 62 | 63 | def test_time_return_value 64 | result = collector.statsd_time(:foo) { 42 } 65 | assert_equal(42, result) 66 | end 67 | 68 | def test_timing 69 | collector.expects(:__statsd_publish).with(:timing, :foo, 5, 0.5) 70 | collector.expects(:__statsd_publish).with(:timing, :bar, 15, nil) 71 | collector.expects(:__statsd_publish).with(:timing, :baz, 1, nil) 72 | collector.statsd_timing(:foo, 5, 0.5) 73 | collector.statsd_timing(:bar, 15) 74 | collector.statsd_timing(:baz) 75 | end 76 | 77 | def test___statsd_publish 78 | ::ActiveSupport::Notifications.expects(:instrument).with("count.statsd", { :key => :foo }) 79 | collector.__statsd_publish(:count, :foo) 80 | end 81 | 82 | def test___statsd_publish_with_value 83 | ::ActiveSupport::Notifications.expects(:instrument).with("count.statsd", { :key => :foo, :value => 32 }) 84 | collector.__statsd_publish(:count, :foo, 32) 85 | end 86 | 87 | def test___statsd_publish_with_sample_rate 88 | ::ActiveSupport::Notifications.expects(:instrument).with("count.statsd", { :key => :foo, :sample_rate => 0.5 }) 89 | ::ActiveSupport::Notifications.expects(:instrument).with("count.statsd", { :key => :bar, :value => 23, :sample_rate => 0.5 }) 90 | collector.__statsd_publish(:count, :foo, nil, 0.5) 91 | collector.__statsd_publish(:count, :bar, 23, 0.5) 92 | end 93 | 94 | end 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NSA (National Statsd Agency) 2 | 3 | Listen to Rails `ActiveSupport::Notifications` and deliver to a [Statsd](https://github.com/reinh/statsd) backend. 4 | This gem also supports writing your own custom collectors. 5 | 6 | [![Gem Version](https://badge.fury.io/rb/nsa.svg)](https://badge.fury.io/rb/nsa) 7 | [![Build Status](https://travis-ci.org/localshred/nsa.svg)](https://travis-ci.org/localshred/nsa) 8 | 9 | ## Installation 10 | 11 | Add this line to your application's Gemfile: 12 | 13 | ```ruby 14 | gem "nsa" 15 | ``` 16 | 17 | And then execute: 18 | 19 | $ bundle 20 | 21 | Or install it yourself as: 22 | 23 | $ gem install nsa 24 | 25 | ## Usage 26 | 27 | NSA comes packaged with collectors for ActionController, ActiveRecord, ActiveSupport Caching, 28 | and Sidekiq. 29 | 30 | To use this gem, simply get a reference to a statsd backend, then indicate which 31 | collectors you'd like to run. Each `collect` method specifies a Collector to use 32 | and the additional key namespace. 33 | 34 | ```ruby 35 | statsd = ::Statsd.new(ENV["STATSD_HOST"], ENV["STATSD_PORT"]) 36 | application_name = ::Rails.application.class.parent_name.underscore 37 | application_env = ENV["PLATFORM_ENV"] || ::Rails.env 38 | statsd.namespace = [ application_name, application_env ].join(".") 39 | 40 | ::NSA.inform_statsd(statsd) do |informant| 41 | # Load :action_controller collector with a key prefix of :web 42 | informant.collect(:action_controller, :web) 43 | informant.collect(:active_record, :db) 44 | informant.collect(:cache, :cache) 45 | informant.collect(:sidekiq, :sidekiq) 46 | end 47 | ``` 48 | 49 | ## Built-in Collectors 50 | 51 | ### `:action_controller` 52 | 53 | Listens to: `process_action.action_controller` 54 | 55 | Metrics recorded: 56 | 57 | + Timing: `{ns}.{prefix}.{controller}.{action}.{format}.total_duration` 58 | + Timing: `{ns}.{prefix}.{controller}.{action}.{format}.db_time` 59 | + Timing: `{ns}.{prefix}.{controller}.{action}.{format}.view_time` 60 | + Increment: `{ns}.{prefix}.{controller}.{action}.{format}.status.{status_code}` 61 | 62 | ### `:active_record` 63 | 64 | Listens to: `sql.active_record` 65 | 66 | Metrics recorded: 67 | 68 | + Timing: `{ns}.{prefix}.tables.{table_name}.queries.delete.duration` 69 | + Timing: `{ns}.{prefix}.tables.{table_name}.queries.insert.duration` 70 | + Timing: `{ns}.{prefix}.tables.{table_name}.queries.select.duration` 71 | + Timing: `{ns}.{prefix}.tables.{table_name}.queries.update.duration` 72 | 73 | ### `:active_support_cache` 74 | 75 | Listens to: `cache_*.active_suppport` 76 | 77 | Metrics recorded: 78 | 79 | + Timing: `{ns}.{prefix}.delete.duration` 80 | + Timing: `{ns}.{prefix}.exist?.duration` 81 | + Timing: `{ns}.{prefix}.fetch_hit.duration` 82 | + Timing: `{ns}.{prefix}.generate.duration` 83 | + Timing: `{ns}.{prefix}.read_hit.duration` 84 | + Timing: `{ns}.{prefix}.read_miss.duration` 85 | + Timing: `{ns}.{prefix}.read_miss.duration` 86 | 87 | ### `:sidekiq` 88 | 89 | Listens to: Sidekiq middleware, run before each job that is processed 90 | 91 | Metrics recorded: 92 | 93 | + Time: `{ns}.{prefix}.{WorkerName}.processing_time` 94 | + Increment: `{ns}.{prefix}.{WorkerName}.success` 95 | + Increment: `{ns}.{prefix}.{WorkerName}.failure` 96 | + Gauge: `{ns}.{prefix}.queues.{queue_name}.enqueued` 97 | + Gauge: `{ns}.{prefix}.queues.{queue_name}.latency` 98 | + Gauge: `{ns}.{prefix}.dead_size` 99 | + Gauge: `{ns}.{prefix}.enqueued` 100 | + Gauge: `{ns}.{prefix}.failed` 101 | + Gauge: `{ns}.{prefix}.processed` 102 | + Gauge: `{ns}.{prefix}.processes_size` 103 | + Gauge: `{ns}.{prefix}.retry_size` 104 | + Gauge: `{ns}.{prefix}.scheduled_size` 105 | + Gauge: `{ns}.{prefix}.workers_size` 106 | 107 | ## Writing your own collector 108 | 109 | Writing your own collector is very simple. To take advantage of the keyspace handling you must: 110 | 111 | 1. Create an object/module which responds to `collect`, taking the `key_prefix` as its only argument. 112 | 2. Include or extend your class/module with `NSA::Statsd::Publisher` or `NSA::Statsd::Publisher`. 113 | 3. Call any of the `statsd_*` prefixed methods provided by the included Publisher: 114 | 115 | __`Publisher` methods:__ 116 | 117 | + `statsd_count(key, value = 1, sample_rate = nil)` 118 | + `statsd_decrement(key, sample_rate = nil)` 119 | + `statsd_gauge(key, value = 1, sample_rate = nil)` 120 | + `statsd_increment(key, sample_rate = nil)` 121 | + `statsd_set(key, value = 1, sample_rate = nil)` 122 | + `statsd_time(key, sample_rate = nil, &block)` 123 | + `statsd_timing(key, value = 1, sample_rate = nil)` 124 | 125 | __`AsyncPublisher` methods:__ 126 | 127 | + `async_statsd_count(key, sample_rate = nil, &block)` 128 | + `async_statsd_gauge(key, sample_rate = nil, &block)` 129 | + `async_statsd_set(key, sample_rate = nil, &block)` 130 | + `async_statsd_time(key, sample_rate = nil, &block)` 131 | + `async_statsd_timing(key, sample_rate = nil, &block)` 132 | 133 | ___Note:___ When using the `AsyncPublisher`, the value is derived from the block. This is useful 134 | when the value is not near at hand and has a relatively high cost to compute (e.g. db query) 135 | and you don't want your current thread to wait. 136 | 137 | For example, first define your collector. Our (very naive) example will write 138 | a gauge metric every 10 seconds of the User count in the db. 139 | 140 | ```ruby 141 | # Publishing User.count gauge using a collector 142 | module UsersCollector 143 | extend ::NSA::Statsd::Publisher 144 | 145 | def self.collect(key_prefix) 146 | loop do 147 | statsd_gauge("count", ::User.count) 148 | sleep 10 # don't do this, obvi 149 | end 150 | end 151 | end 152 | ``` 153 | 154 | Then let the informant know about it in some initializer: 155 | 156 | ```ruby 157 | # file: config/initializers/statsd.rb 158 | 159 | # $statsd = 160 | NSA.inform_statsd($statsd) do |informant| 161 | # ... 162 | informant.collect(UserCollector, :users) 163 | end 164 | ``` 165 | 166 | You could also implement the provided example not as a Collector, but using 167 | `AsyncPublisher` directly in your ActiveRecord model: 168 | 169 | ```ruby 170 | # Publishing User.count gauge using AsyncPublisher methods 171 | class User < ActiveRecord::Base 172 | include NSA::Statsd::AsyncPublisher 173 | 174 | after_commit :write_count_gauge, :on => [ :create, :destroy ] 175 | 176 | # ... 177 | 178 | private 179 | 180 | def write_count_gauge 181 | async_statsd_gauge("models.User.all.count") { ::User.count } 182 | end 183 | 184 | end 185 | ``` 186 | 187 | Using this technique, publishing the `User.count` stat gauge will not hold up 188 | the thread responsible for creating the record (and processing more callbacks). 189 | 190 | ## Development 191 | 192 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 193 | 194 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 195 | 196 | ## Contributing 197 | 198 | Bug reports and pull requests are welcome on GitHub at https://github.com/localshred/nsa. 199 | 200 | --------------------------------------------------------------------------------