├── attributes ├── nagios_plugins.rb ├── handlers.rb └── default.rb ├── .gitignore ├── test └── integration │ ├── data_bags │ ├── sensu_checks │ │ ├── ssh.json │ │ ├── fs_writable.json │ │ └── ntp.json │ └── sensu │ │ └── ssl.json │ └── default │ └── bats │ └── default.bats ├── Gemfile ├── Cheffile ├── metadata.rb ├── recipes ├── _nagios_plugins.rb ├── _sudo.rb ├── _redis.rb ├── _rabbitmq.rb ├── master.rb ├── rabbitmq.rb ├── _filters.rb ├── _statsd.rb ├── haproxy.rb ├── _system_profile.rb ├── _nagios_perfdata.rb ├── redis.rb ├── _haproxy.rb ├── _pagerduty_handler.rb ├── _extensions.rb ├── _chef_node_handler.rb ├── _master_search.rb ├── default.rb ├── _worker.rb └── _graphite_handler.rb ├── files └── default │ ├── plugins │ ├── check-fs-writable.rb │ ├── check-mtime.rb │ ├── check-tail.rb │ ├── check-banner.rb │ ├── redis-metrics.rb │ ├── check-haproxy.rb │ ├── check-log.rb │ ├── check-http.rb │ ├── rabbitmq-overview-metrics.rb │ └── check-procs.rb │ ├── extensions │ ├── nagios_perfdata.rb │ ├── statsd.rb │ └── system_profile.rb │ └── handlers │ ├── pagerduty.rb │ └── chef_node.rb ├── .kitchen.yml └── README.md /attributes/nagios_plugins.rb: -------------------------------------------------------------------------------- 1 | default["monitor"]["nagios_plugin_packages"] = ["nagios-plugins"] 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.lock 2 | .cache 3 | .bundle 4 | .librarian 5 | .kitchen/ 6 | .kitchen.local.yml* 7 | tmp/ 8 | vendor/ 9 | -------------------------------------------------------------------------------- /attributes/handlers.rb: -------------------------------------------------------------------------------- 1 | default["monitor"]["pagerduty_api_key"] = "" 2 | 3 | default["monitor"]["graphite_address"] = nil 4 | default["monitor"]["graphite_port"] = nil 5 | -------------------------------------------------------------------------------- /test/integration/data_bags/sensu_checks/ssh.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "ssh", 3 | "command": "check-banner.rb", 4 | "subscribers": [ 5 | "all" 6 | ], 7 | "interval": 60 8 | } 9 | -------------------------------------------------------------------------------- /test/integration/data_bags/sensu_checks/fs_writable.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "fs_writable", 3 | "command": "check-fs-writable.rb -d /tmp", 4 | "subscribers": [ 5 | "all" 6 | ], 7 | "interval": 30 8 | } 9 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "test-kitchen", ">= 1.0.0.alpha.7" 4 | gem "librarian-chef", ">= 0.0.1" 5 | 6 | group :docker do 7 | gem "kitchen-docker", ">= 0.8.0" 8 | end 9 | 10 | group :vagrant do 11 | gem "kitchen-vagrant" 12 | end 13 | -------------------------------------------------------------------------------- /test/integration/data_bags/sensu_checks/ntp.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "ntp", 3 | "type": "metric", 4 | "command": "/usr/lib/nagios/plugins/check_ntp -H time.nrc.ca", 5 | "subscribers": [ 6 | "all" 7 | ], 8 | "interval": 60, 9 | "handlers": ["graphite_perfdata"] 10 | } 11 | -------------------------------------------------------------------------------- /Cheffile: -------------------------------------------------------------------------------- 1 | site 'http://community.opscode.com/api/v1' 2 | 3 | cookbook "apt" 4 | cookbook "yum" 5 | cookbook "logrotate" 6 | 7 | cookbook "sensu", 8 | :git => "https://github.com/sensu/sensu-chef.git" 9 | 10 | cookbook "monitor", 11 | :path => "./" 12 | 13 | cookbook "haproxy", "1.4.0" 14 | 15 | cookbook "python", 16 | :git => "https://github.com/hw-cookbooks/python.git", 17 | :ref => "ohai" 18 | 19 | cookbook "graphite", 20 | :git => "https://github.com/hw-cookbooks/graphite.git", 21 | :ref => "v0.4.6" 22 | -------------------------------------------------------------------------------- /metadata.rb: -------------------------------------------------------------------------------- 1 | name "monitor" 2 | maintainer "Sean Porter Consulting" 3 | maintainer_email "portertech@gmail.com" 4 | license "Apache 2.0" 5 | description "A cookbook for monitoring services, using Sensu, a monitoring framework." 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 7 | version "0.0.6" 8 | 9 | %w[ 10 | ubuntu 11 | debian 12 | centos 13 | redhat 14 | fedora 15 | ].each do |os| 16 | supports os 17 | end 18 | 19 | depends "sensu" 20 | depends "sudo" 21 | depends "uchiwa" 22 | -------------------------------------------------------------------------------- /attributes/default.rb: -------------------------------------------------------------------------------- 1 | override["sensu"]["use_embedded_ruby"] = true 2 | 3 | default["monitor"]["master_address"] = nil 4 | 5 | default["monitor"]["environment_aware_search"] = false 6 | default["monitor"]["use_local_ipv4"] = false 7 | 8 | default["monitor"]["additional_client_attributes"] = Mash.new 9 | 10 | default["monitor"]["use_nagios_plugins"] = false 11 | default["monitor"]["use_system_profile"] = false 12 | default["monitor"]["use_statsd_input"] = false 13 | 14 | default["monitor"]["sudo_commands"] = Array.new 15 | 16 | default["monitor"]["default_handlers"] = ["debug"] 17 | default["monitor"]["metric_handlers"] = ["debug"] 18 | 19 | default["monitor"]["client_extension_dir"] = "/etc/sensu/extensions/client" 20 | default["monitor"]["server_extension_dir"] = "/etc/sensu/extensions/server" 21 | -------------------------------------------------------------------------------- /recipes/_nagios_plugins.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _nagios_plugins 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | node["monitor"]["nagios_plugin_packages"].each do |package_name| 21 | package package_name 22 | end 23 | -------------------------------------------------------------------------------- /recipes/_sudo.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _sudo 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "sudo" 21 | 22 | sudo "sensu" do 23 | user "sensu" 24 | runas "root" 25 | commands node["monitor"]["sudo_commands"] 26 | host "ALL" 27 | nopasswd true 28 | end 29 | -------------------------------------------------------------------------------- /recipes/_redis.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _redis 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::default" 21 | 22 | sensu_gem "redis" 23 | 24 | cookbook_file "/etc/sensu/plugins/redis-metrics.rb" do 25 | source "plugins/redis-metrics.rb" 26 | mode 0755 27 | end 28 | -------------------------------------------------------------------------------- /recipes/_rabbitmq.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _rabbitmq 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::default" 21 | 22 | sensu_gem "carrot-top" 23 | 24 | cookbook_file "/etc/sensu/plugins/rabbitmq-overview-metrics.rb" do 25 | source "plugins/rabbitmq-overview-metrics.rb" 26 | mode 0755 27 | end 28 | -------------------------------------------------------------------------------- /recipes/master.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: master 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "sensu::rabbitmq" 21 | include_recipe "sensu::redis" 22 | 23 | include_recipe "monitor::_worker" 24 | 25 | include_recipe "sensu::api_service" 26 | include_recipe "uchiwa" 27 | 28 | include_recipe "monitor::default" 29 | -------------------------------------------------------------------------------- /recipes/rabbitmq.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: rabbitmq 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::_rabbitmq" 21 | 22 | sensu_check "rabbitmq_overview_metrics" do 23 | type "metric" 24 | command "rabbitmq-overview-metrics.rb" 25 | handlers ["metrics"] 26 | standalone true 27 | interval 30 28 | end 29 | -------------------------------------------------------------------------------- /recipes/_filters.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _filters 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | sensu_filter "actions" do 21 | attributes(:action => "eval: %w[create resolve].include? value.to_s") 22 | end 23 | 24 | sensu_filter "keepalives" do 25 | attributes( 26 | :check => { 27 | :name => "keepalive" 28 | } 29 | ) 30 | end 31 | -------------------------------------------------------------------------------- /recipes/_statsd.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _statsd 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::_extensions" 21 | 22 | cookbook_file File.join(node["monitor"]["client_extension_dir"], "statsd.rb") do 23 | source "extensions/statsd.rb" 24 | mode 0755 25 | notifies :create, "ruby_block[sensu_service_trigger]", :immediately 26 | end 27 | -------------------------------------------------------------------------------- /recipes/haproxy.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: haproxy 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::_haproxy" 21 | 22 | sensu_check "haproxy_services" do 23 | command "sudo check-haproxy.rb -s :::haproxy_services::: -w :::haproxy_warning|75::: -c :::haproxy_critical|50:::" 24 | handlers ["default"] 25 | standalone true 26 | interval 30 27 | end 28 | -------------------------------------------------------------------------------- /recipes/_system_profile.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _system_profile 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::_extensions" 21 | 22 | cookbook_file File.join(node["monitor"]["client_extension_dir"], "system_profile.rb") do 23 | source "extensions/system_profile.rb" 24 | mode 0755 25 | notifies :create, "ruby_block[sensu_service_trigger]", :immediately 26 | end 27 | -------------------------------------------------------------------------------- /recipes/_nagios_perfdata.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _nagios_perfdata 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::_extensions" 21 | 22 | cookbook_file File.join(node["monitor"]["server_extension_dir"], "nagios_perfdata.rb") do 23 | source "extensions/nagios_perfdata.rb" 24 | mode 0755 25 | notifies :create, "ruby_block[sensu_service_trigger]", :immediately 26 | end 27 | -------------------------------------------------------------------------------- /files/default/plugins/check-fs-writable.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Check Filesystem Writability Plugin 4 | # === 5 | # 6 | # This plugin checks that a filesystem is writable. Useful for checking for stale NFS mounts. 7 | 8 | require 'rubygems' if RUBY_VERSION < '1.9.0' 9 | require 'sensu-plugin/check/cli' 10 | require 'tempfile' 11 | 12 | class CheckFSWritable < Sensu::Plugin::Check::CLI 13 | 14 | option :dir, 15 | :description => 'Directory to check for writability', 16 | :short => '-d DIRECTORY', 17 | :long => '--directory DIRECTORY' 18 | 19 | def run 20 | unknown 'No directory specified' unless config[:dir] 21 | critical "#{config[:dir]} does not exist " if !File.directory?(config[:dir]) 22 | file = Tempfile.new('.sensu', config[:dir]) 23 | begin 24 | file.write("mops") or critical 'Could not write to filesystem' 25 | file.read or critical 'Could not read from filesystem' 26 | ensure 27 | file.close 28 | file.unlink 29 | end 30 | ok "#{config[:dir]} is OK" 31 | end 32 | 33 | end 34 | -------------------------------------------------------------------------------- /recipes/redis.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: redis 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::_redis" 21 | 22 | sensu_check "redis_process" do 23 | command "check-procs.rb -p redis-server -w 2 -c 3 -C 1" 24 | handlers ["default"] 25 | standalone true 26 | interval 30 27 | end 28 | 29 | sensu_check "redis_metrics" do 30 | type "metric" 31 | command "redis-metrics.rb" 32 | handlers ["metrics"] 33 | standalone true 34 | interval 30 35 | end 36 | -------------------------------------------------------------------------------- /recipes/_haproxy.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _haproxy 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::default" 21 | 22 | sensu_gem "haproxy" 23 | 24 | plugin_path = "/etc/sensu/plugins/check-haproxy.rb" 25 | 26 | cookbook_file plugin_path do 27 | source "plugins/check-haproxy.rb" 28 | mode 0755 29 | end 30 | 31 | node.override["monitor"]["sudo_commands"] = 32 | node["monitor"]["sudo_commands"] + [plugin_path] 33 | 34 | include_recipe "monitor::_sudo" 35 | -------------------------------------------------------------------------------- /recipes/_pagerduty_handler.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _pagerduty_handler 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | sensu_gem "redphone" 21 | 22 | cookbook_file "/etc/sensu/handlers/pagerduty.rb" do 23 | source "handlers/pagerduty.rb" 24 | mode 0755 25 | end 26 | 27 | sensu_snippet "pagerduty" do 28 | content(:api_key => node["monitor"]["pagerduty_api_key"]) 29 | end 30 | 31 | include_recipe "monitor::_filters" 32 | 33 | sensu_handler "pagerduty" do 34 | type "pipe" 35 | command "pagerduty.rb" 36 | filters ["actions"] 37 | end 38 | -------------------------------------------------------------------------------- /test/integration/default/bats/default.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | @test "should have embedded ruby" { 4 | [ -x "/opt/sensu/embedded/bin/ruby" ] 5 | } 6 | 7 | @test "should have rabbitmq running" { 8 | [ "$(ps aux | grep rabbitmq-server | grep -v grep)" ] 9 | } 10 | 11 | @test "rabbitmq should be listening for connections" { 12 | [ "$(netstat -plant | grep beam)" ] 13 | } 14 | 15 | @test "should have redis-server running" { 16 | [ "$(ps aux | grep redis-server | grep -v grep)" ] 17 | } 18 | 19 | @test "redis-server should be listening for connections" { 20 | [ "$(netstat -plant | grep redis-server)" ] 21 | } 22 | 23 | @test "should have sensu server running" { 24 | [ "$(ps aux | grep sensu-server | grep -v grep)" ] 25 | } 26 | 27 | @test "should have sensu api running" { 28 | [ "$(ps aux | grep sensu-api | grep -v grep)" ] 29 | } 30 | 31 | @test "should have sensu dashboard running" { 32 | [ "$(ps aux | grep sensu-dashboard | grep -v grep)" ] 33 | } 34 | 35 | @test "should have sensu client running" { 36 | [ "$(ps aux | grep sensu-client | grep -v grep)" ] 37 | } 38 | 39 | @test "should have a ssh check definition" { 40 | [ -e "/etc/sensu/conf.d/checks/ssh.json" ] 41 | } 42 | -------------------------------------------------------------------------------- /files/default/extensions/nagios_perfdata.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Nagios performance data to Graphite plain text mutator extension. 3 | # === 4 | # 5 | # Copyright 2013 Heavy Water Operations, LLC. 6 | # 7 | # Released under the same terms as Sensu (the MIT license); see LICENSE 8 | # for details. 9 | 10 | module Sensu 11 | module Extension 12 | class NagiosPerfData < Mutator 13 | def name 14 | 'nagios_perfdata' 15 | end 16 | 17 | def description 18 | 'converts nagios performance data to graphite plain text' 19 | end 20 | 21 | def run(event) 22 | result = [] 23 | client = event[:client] 24 | check = event[:check] 25 | 26 | # https://www.nagios-plugins.org/doc/guidelines.html#AEN200 27 | perfdata = check[:output].split('|').last.strip 28 | 29 | perfdata.split(/\s+/).each do |data| 30 | # label=value[UOM];[warn];[crit];[min];[max] 31 | label, value = data.split('=') 32 | 33 | name = label.strip.gsub(/\W/, '_') 34 | measurement = value.strip.split(';').first.gsub(/[^-\d\.]/, '') 35 | 36 | path = [client[:name], check[:name], name].join('.') 37 | 38 | result << [path, measurement, check[:executed]].join("\t") 39 | end 40 | yield(result.join("\n") + "\n", 0) 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /recipes/_extensions.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _extensions 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | %w[ 21 | client 22 | server 23 | ].each do |service| 24 | extension_dir = node["monitor"]["#{service}_extension_dir"] 25 | 26 | directory extension_dir do 27 | recursive true 28 | owner "root" 29 | group "sensu" 30 | mode 0750 31 | end 32 | 33 | config_path = case node.platform_family 34 | when "rhel", "fedora" 35 | "/etc/sysconfig/sensu-#{service}" 36 | else 37 | "/etc/default/sensu-#{service}" 38 | end 39 | 40 | file config_path do 41 | owner "root" 42 | group "root" 43 | mode 0744 44 | content "EXTENSION_DIR=#{extension_dir}" 45 | notifies :create, "ruby_block[sensu_service_trigger]", :immediately 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /recipes/_chef_node_handler.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _chef_node_handler 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | sensu_gem "spice" do 21 | version "1.0.6" 22 | end 23 | 24 | sensu_gem "rest-client" 25 | 26 | handler_path = "/etc/sensu/handlers/chef_node.rb" 27 | 28 | cookbook_file handler_path do 29 | source "handlers/chef_node.rb" 30 | mode 0755 31 | end 32 | 33 | node.override['monitor']['sudo_commands'] = 34 | node['monitor']['sudo_commands'] + [handler_path] 35 | 36 | include_recipe "monitor::_sudo" 37 | 38 | sensu_snippet "chef" do 39 | content( 40 | :server_url => Chef::Config[:chef_server_url], 41 | :client_name => Chef::Config[:node_name], 42 | :client_key => Chef::Config[:client_key], 43 | :verify_ssl => false 44 | ) 45 | end 46 | 47 | include_recipe "monitor::_filters" 48 | 49 | sensu_handler "chef_node" do 50 | type "pipe" 51 | command "sudo chef_node.rb" 52 | filters ["keepalives"] 53 | end 54 | -------------------------------------------------------------------------------- /recipes/_master_search.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _master_search 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 16 | # implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | # 20 | 21 | ip_type = node["monitor"]["use_local_ipv4"] ? "local_ipv4" : "public_ipv4" 22 | master_address = node["monitor"]["master_address"] 23 | 24 | case 25 | when Chef::Config[:solo] 26 | master_address ||= "localhost" 27 | when master_address.nil? 28 | if node["recipes"].include?("monitor::master") 29 | master_address = "localhost" 30 | else 31 | master_node = case 32 | when node["monitor"]["environment_aware_search"] 33 | search(:node, "chef_environment:#{node.chef_environment} AND recipes:monitor\\:\\:master").first 34 | else 35 | search(:node, "recipes:monitor\\:\\:master").first 36 | end 37 | 38 | master_address = case 39 | when master_node.has_key?("cloud") 40 | master_node["cloud"][ip_type] || master_node["ipaddress"] 41 | else 42 | master_node["ipaddress"] 43 | end 44 | end 45 | end 46 | 47 | node.override["sensu"]["rabbitmq"]["host"] = master_address 48 | node.override["sensu"]["redis"]["host"] = master_address 49 | node.override["sensu"]["api"]["host"] = master_address 50 | -------------------------------------------------------------------------------- /files/default/plugins/check-mtime.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Checks a file's mtime 4 | # === 5 | # 6 | # DESCRIPTION: 7 | # This plugin checks a given file's modified time. 8 | # 9 | # OUTPUT: 10 | # plain-text 11 | # 12 | # PLATFORMS: 13 | # linux 14 | # bsd 15 | # 16 | # DEPENDENCIES: 17 | # sensu-plugin Ruby gem 18 | # 19 | # Released under the same terms as Sensu (the MIT license); see LICENSE 20 | # for details. 21 | 22 | require 'rubygems' if RUBY_VERSION < '1.9.0' 23 | require 'sensu-plugin/check/cli' 24 | require 'fileutils' 25 | 26 | class Mtime < Sensu::Plugin::Check::CLI 27 | 28 | option :file, 29 | :description => 'File to check last modified time', 30 | :short => '-f FILE', 31 | :long => '--file FILE' 32 | 33 | option :warn_age, 34 | :description => 'Warn if mtime greater than provided age in seconds', 35 | :short => '-w SECONDS', 36 | :long => '--warn SECONDS' 37 | 38 | option :critical_age, 39 | :description => 'Critical if mtime greater than provided age in seconds', 40 | :short => '-c SECONDS', 41 | :long => '--critical SECONDS' 42 | 43 | def run_check(type, age) 44 | to_check = config["#{type}_age".to_sym].to_i 45 | if(to_check > 0 && age >= to_check) 46 | send(type, "file is #{age - to_check} seconds past #{type}") 47 | end 48 | end 49 | 50 | def run 51 | unknown 'No file specified' unless config[:file] 52 | unknown 'No warn or critical age specified' unless config[:warn_age] || config[:critical_age] 53 | if(File.exists?(config[:file])) 54 | age = Time.now.to_i - File.mtime(config[:file]).to_i 55 | run_check(:critical, age) || run_check(:warning, age) || ok("file is #{age} seconds old") 56 | else 57 | critical 'file not found' 58 | end 59 | end 60 | 61 | end 62 | -------------------------------------------------------------------------------- /.kitchen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | driver_plugin: vagrant 3 | driver_config: 4 | require_chef_omnibus: true 5 | 6 | platforms: 7 | - name: ubuntu-13.04 8 | driver_config: 9 | box: opscode-ubuntu-13.04 10 | box_url: https://opscode-vm-bento.s3.amazonaws.com/vagrant/opscode_ubuntu-13.04_provisionerless.box 11 | customize: 12 | memory: 1024 13 | network: 14 | - ["forwarded_port", {guest: 80, host: 8080}] 15 | - ["forwarded_port", {guest: 8080, host: 8081}] 16 | run_list: 17 | - recipe[apt] 18 | 19 | - name: centos-6.4 20 | driver_config: 21 | box: opscode-centos-6.4 22 | box_url: https://opscode-vm-bento.s3.amazonaws.com/vagrant/opscode_centos-6.4_provisionerless.box 23 | customize: 24 | memory: 1024 25 | network: 26 | - ["forwarded_port", {guest: 80, host: 8080}] 27 | - ["forwarded_port", {guest: 8080, host: 8081}] 28 | run_list: 29 | - recipe[yum] 30 | 31 | suites: 32 | - name: default 33 | run_list: 34 | - recipe[haproxy] 35 | - recipe[graphite] 36 | - recipe[monitor::master] 37 | - recipe[monitor::haproxy] 38 | - recipe[monitor::redis] 39 | - recipe[monitor::rabbitmq] 40 | attributes: 41 | authorization: 42 | sudo: 43 | users: ["vagrant", "kitchen"] 44 | passwordless: true 45 | include_sudoers_d: true 46 | haproxy: 47 | enable_stats_socket: true 48 | apache: 49 | listen_ports: [8080] 50 | graphite: 51 | listen_port: 8080 52 | carbon: 53 | service_type: "init" 54 | sensu: 55 | dashboard: 56 | port: 4000 57 | monitor: 58 | use_nagios_plugins: true 59 | use_system_profile: true 60 | use_statsd_input: true 61 | metric_handlers: ["graphite"] 62 | additional_client_attributes: 63 | haproxy_services: "servers-http" 64 | -------------------------------------------------------------------------------- /files/default/handlers/pagerduty.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # This handler creates and resolves PagerDuty incidents, refreshing 4 | # stale incident details every 30 minutes 5 | # 6 | # Copyright 2011 Sonian, Inc 7 | # 8 | # Released under the same terms as Sensu (the MIT license); see LICENSE 9 | # for details. 10 | 11 | require 'rubygems' if RUBY_VERSION < '1.9.0' 12 | require 'sensu-handler' 13 | require 'redphone/pagerduty' 14 | 15 | class Pagerduty < Sensu::Handler 16 | 17 | def incident_key 18 | @event['client']['name'] + '/' + @event['check']['name'] 19 | end 20 | 21 | def handle 22 | description = @event['check']['notification'] 23 | description ||= [@event['client']['name'], @event['check']['name'], @event['check']['output']].join(' : ') 24 | begin 25 | timeout(10) do 26 | response = case @event['action'] 27 | when 'create' 28 | Redphone::Pagerduty.trigger_incident( 29 | :service_key => settings['pagerduty']['api_key'], 30 | :incident_key => incident_key, 31 | :description => description, 32 | :details => @event 33 | ) 34 | when 'resolve' 35 | Redphone::Pagerduty.resolve_incident( 36 | :service_key => settings['pagerduty']['api_key'], 37 | :incident_key => incident_key 38 | ) 39 | end 40 | if response['status'] == 'success' 41 | puts 'pagerduty -- ' + @event['action'].capitalize + 'd incident -- ' + incident_key 42 | else 43 | puts 'pagerduty -- failed to ' + @event['action'] + ' incident -- ' + incident_key 44 | end 45 | end 46 | rescue Timeout::Error 47 | puts 'pagerduty -- timed out while attempting to ' + @event['action'] + ' a incident -- ' + incident_key 48 | end 49 | end 50 | 51 | end 52 | -------------------------------------------------------------------------------- /files/default/plugins/check-tail.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Checks a file's tail for a pattern 4 | # === 5 | # 6 | # DESCRIPTION: 7 | # This plugin checks the tail of a file for a given patten and sends 8 | # critical (or optionally warning) message if found 9 | # 10 | # OUTPUT: 11 | # plain-text 12 | # 13 | # PLATFORMS: 14 | # linux 15 | # bsd 16 | # 17 | # DEPENDENCIES: 18 | # sensu-plugin Ruby gem 19 | # 20 | # Released under the same terms as Sensu (the MIT license); see LICENSE 21 | # for details. 22 | 23 | require 'rubygems' if RUBY_VERSION < '1.9.0' 24 | require 'sensu-plugin/check/cli' 25 | require 'fileutils' 26 | 27 | class Tail < Sensu::Plugin::Check::CLI 28 | 29 | option :file, 30 | :description => "Path to file", 31 | :short => '-f FILE', 32 | :long => '--file FILE' 33 | 34 | option :pattern, 35 | :description => "Pattern to search for", 36 | :short => '-P PAT', 37 | :long => '--pattern PAT' 38 | 39 | option :lines, 40 | :description => "Number of lines to tail", 41 | :short => '-l LINES', 42 | :long => '--lines LINES' 43 | 44 | option :warn_only, 45 | :description => "Warn instead of critical on match", 46 | :short => '-w', 47 | :long => '--warn-only', 48 | :boolean => true 49 | 50 | def tail_file 51 | `tail #{config[:file]} -n #{config[:lines] || 1}` 52 | end 53 | 54 | def pattern_match? 55 | !!tail_file.match(config[:pattern]) 56 | end 57 | 58 | def run 59 | unknown "No log file specified" unless config[:file] 60 | unknown "No pattern specified" unless config[:pattern] 61 | if File.exists?(config[:file]) 62 | if pattern_match? 63 | send( 64 | config[:warn_only] ? :warning : :critical, 65 | "Pattern matched: #{config[:pattern]}" 66 | ) 67 | else 68 | ok "No matches found" 69 | end 70 | else 71 | critical 'File not found' 72 | end 73 | end 74 | 75 | end 76 | -------------------------------------------------------------------------------- /recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: default 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::_master_search" 21 | 22 | include_recipe "sensu::default" 23 | 24 | ip_type = node["monitor"]["use_local_ipv4"] ? "local_ipv4" : "public_ipv4" 25 | 26 | client_attributes = node["monitor"]["additional_client_attributes"].to_hash 27 | 28 | sensu_client node.name do 29 | if node.has_key?("cloud") 30 | address node["cloud"][ip_type] || node["ipaddress"] 31 | else 32 | address node["ipaddress"] 33 | end 34 | subscriptions node["roles"] + ["all"] 35 | additional client_attributes 36 | end 37 | 38 | %w[ 39 | check-procs.rb 40 | check-banner.rb 41 | check-http.rb 42 | check-log.rb 43 | check-mtime.rb 44 | check-tail.rb 45 | check-fs-writable.rb 46 | ].each do |default_plugin| 47 | cookbook_file "/etc/sensu/plugins/#{default_plugin}" do 48 | source "plugins/#{default_plugin}" 49 | mode 0755 50 | end 51 | end 52 | 53 | if node["monitor"]["use_nagios_plugins"] 54 | include_recipe "monitor::_nagios_plugins" 55 | end 56 | 57 | if node["monitor"]["use_system_profile"] 58 | include_recipe "monitor::_system_profile" 59 | end 60 | 61 | if node["monitor"]["use_statsd_input"] 62 | include_recipe "monitor::_statsd" 63 | end 64 | 65 | include_recipe "sensu::client_service" 66 | -------------------------------------------------------------------------------- /files/default/plugins/check-banner.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Check Banner 4 | # === 5 | # 6 | # Connect to a TCP port, read one line, test it against a pattern. 7 | # Useful for SSH, ZooKeeper, etc. 8 | # 9 | # Copyright 2013 Sonian, Inc 10 | # 11 | # Released under the same terms as Sensu (the MIT license); see LICENSE 12 | # for details. 13 | 14 | require 'rubygems' if RUBY_VERSION < '1.9.0' 15 | require 'sensu-plugin/check/cli' 16 | require 'socket' 17 | require 'timeout' 18 | 19 | class CheckBanner < Sensu::Plugin::Check::CLI 20 | 21 | option :host, 22 | :short => '-H HOSTNAME', 23 | :long => '--hostname HOSTNAME', 24 | :description => 'Host to connect to', 25 | :default => 'localhost' 26 | 27 | option :port, 28 | :short => '-p PORT', 29 | :long => '--port PORT', 30 | :proc => proc {|a| a.to_i }, 31 | :default => 22 32 | 33 | option :write, 34 | :short => '-w STRING', 35 | :long => '--write STRING', 36 | :description => 'write STRING to the socket' 37 | 38 | option :pattern, 39 | :short => '-q PAT', 40 | :long => '--pattern PAT', 41 | :description => 'Pattern to search for', 42 | :default => 'OpenSSH' 43 | 44 | option :timeout, 45 | :short => '-t SECS', 46 | :long => '--timeout SECS', 47 | :description => 'Connection timeout', 48 | :proc => proc {|a| a.to_i }, 49 | :default => 30 50 | 51 | def get_banner 52 | begin 53 | sock = TCPSocket.new(config[:host], config[:port]) 54 | timeout(config[:timeout]) do 55 | sock.puts config[:write] if config[:write] 56 | sock.readline 57 | end 58 | rescue Errno::ECONNREFUSED 59 | critical "Connection refused by #{config[:host]}:#{config[:port]}" 60 | rescue Timeout::Error 61 | critical "Connection or read timed out" 62 | end 63 | end 64 | 65 | def run 66 | banner = get_banner 67 | message banner 68 | banner =~ /#{config[:pattern]}/ ? ok : warning 69 | end 70 | 71 | end 72 | -------------------------------------------------------------------------------- /recipes/_worker.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _worker 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "monitor::_master_search" 21 | 22 | include_recipe "sensu::default" 23 | 24 | sensu_gem "sensu-plugin" do 25 | version node["monitor"]["sensu_plugin_version"] 26 | end 27 | 28 | handlers = node["monitor"]["default_handlers"] + node["monitor"]["metric_handlers"] 29 | handlers.each do |handler_name| 30 | next if handler_name == "debug" 31 | include_recipe "monitor::_#{handler_name}_handler" 32 | end 33 | 34 | sensu_handler "default" do 35 | type "set" 36 | handlers node["monitor"]["default_handlers"] 37 | end 38 | 39 | sensu_handler "metrics" do 40 | type "set" 41 | handlers node["monitor"]["metric_handlers"] 42 | end 43 | 44 | check_definitions = case 45 | when Chef::Config[:solo] 46 | data_bag("sensu_checks").map do |item| 47 | data_bag_item("sensu_checks", item) 48 | end 49 | when Chef::DataBag.list.has_key?("sensu_checks") 50 | search(:sensu_checks, "*:*") 51 | else 52 | Array.new 53 | end 54 | 55 | check_definitions.each do |check| 56 | sensu_check check["id"] do 57 | type check["type"] 58 | command check["command"] 59 | subscribers check["subscribers"] 60 | interval check["interval"] 61 | handlers check["handlers"] 62 | additional check["additional"] 63 | end 64 | end 65 | 66 | include_recipe "sensu::server_service" 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Monitor is a cookbook for monitoring services, using Sensu, a 4 | monitoring framework. The default recipe installs & configures the 5 | Sensu client (monitoring agent), as well as common service check 6 | dependencies. The master recipe installs & configures the Sensu server, 7 | API, Uchiwa (dashboard), & their dependencies (eg. RabbitMQ & Redis). 8 | The remaining recipes are intended to put monitoring checks in place 9 | in order to monitor specific services (eg. `recipe[monitor::redis]`). 10 | 11 | Learn more about Sensu [Here](http://sensuapp.org/docs). 12 | 13 | ### THIS COOKBOOK SERVES AS AN EXAMPLE!!! 14 | 15 | There are many ways to deploy/use Sensu and its dependencies, this 16 | "wrapper" cookbook is opinionated, you may not agree with its approach 17 | and choices. If this cookbook can serve as the base for your 18 | monitoring cookbook, fork it :-) 19 | 20 | ## Requirements 21 | 22 | Cookbooks: 23 | 24 | - [Sensu](http://community.opscode.com/cookbooks/sensu) 25 | - [Uchiwa](http://community.opscode.com/cookbooks/uchiwa) 26 | - [sudo](http://community.opscode.com/cookbooks/sudo) 27 | 28 | ## Attributes 29 | 30 | `node["monitor"]["master_address"]` - Bypass the chef node search and 31 | explicitly set the address to reach the master server. 32 | 33 | `node["monitor"]["environment_aware_search"]` - Defaults to false. 34 | If true, will limit search to the node's chef_environment. 35 | 36 | `node["monitor"]["use_local_ipv4"]` - Defaults to false. If true, 37 | use cloud local\_ipv4 when available instead of public\_ipv4. 38 | 39 | `node["monitor"]["sensu_plugin_version"]` - Sensu Plugin library 40 | version. 41 | 42 | `node["monitor"]["additional_client_attributes"]` - Additional client 43 | attributes to be passed to the sensu_client LWRP. 44 | 45 | `node["monitor"]["default_handlers"]` - Default event handlers. 46 | 47 | `node["monitor"]["metric_handlers"]` - Metric event handlers. 48 | 49 | ## Usage 50 | 51 | Example: To monitor the Redis service running on a Chef node, include 52 | "recipe[monitor::redis]" in its run list. 53 | -------------------------------------------------------------------------------- /recipes/_graphite_handler.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: monitor 3 | # Recipe:: _graphite_handler 4 | # 5 | # Copyright 2013, Sean Porter Consulting 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | graphite_address = node["monitor"]["graphite_address"] 21 | graphite_port = node["monitor"]["graphite_port"] 22 | 23 | ip_type = node["monitor"]["use_local_ipv4"] ? "local_ipv4" : "public_ipv4" 24 | 25 | case 26 | when Chef::Config[:solo] 27 | graphite_address ||= "localhost" 28 | graphite_port ||= 2003 29 | when graphite_address.nil? 30 | graphite_node = case 31 | when node["monitor"]["environment_aware_search"] 32 | search(:node, "chef_environment:#{node.chef_environment} AND recipes:graphite").first 33 | else 34 | search(:node, "recipes:graphite").first 35 | end 36 | 37 | graphite_address = case 38 | when graphite_node.has_key?("cloud") 39 | graphite_node["cloud"][ip_type] || graphite_node["ipaddress"] 40 | else 41 | graphite_node["ipaddress"] 42 | end 43 | 44 | graphite_port = graphite_node["graphite"]["carbon"]["line_receiver_port"] 45 | end 46 | 47 | sensu_handler "graphite" do 48 | type "tcp" 49 | socket( 50 | :host => graphite_address, 51 | :port => graphite_port 52 | ) 53 | mutator "only_check_output" 54 | end 55 | 56 | if node["monitor"]["use_nagios_plugins"] 57 | include_recipe "monitor::_nagios_perfdata" 58 | 59 | sensu_handler "graphite_perfdata" do 60 | type "tcp" 61 | socket( 62 | :host => graphite_address, 63 | :port => graphite_port 64 | ) 65 | mutator "nagios_perfdata" 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /files/default/plugins/redis-metrics.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Push Redis INFO stats into graphite 4 | # === 5 | # 6 | # Copyright 2012 Pete Shima 7 | # Brian Racer 8 | # 9 | # Released under the same terms as Sensu (the MIT license); see LICENSE 10 | # for details. 11 | 12 | require 'rubygems' if RUBY_VERSION < '1.9.0' 13 | require 'sensu-plugin/metric/cli' 14 | require 'redis' 15 | 16 | class Redis2Graphite < Sensu::Plugin::Metric::CLI::Graphite 17 | 18 | # redis.c - sds genRedisInfoString(char *section) 19 | SKIP_KEYS_REGEX = ['gcc_version', 'master_host', 'master_link_status', 20 | 'master_port', 'mem_allocator', 'multiplexing_api', 'process_id', 21 | 'redis_git_dirty', 'redis_git_sha1', 'redis_version', '^role', 22 | 'run_id', '^slave', 'used_memory_human', 'used_memory_peak_human'] 23 | 24 | option :host, 25 | :short => "-h HOST", 26 | :long => "--host HOST", 27 | :description => "Redis Host to connect to", 28 | :default => '127.0.0.1' 29 | 30 | option :port, 31 | :short => "-p PORT", 32 | :long => "--port PORT", 33 | :description => "Redis Port to connect to", 34 | :proc => proc {|p| p.to_i }, 35 | :default => 6379 36 | 37 | option :scheme, 38 | :description => "Metric naming scheme, text to prepend to metric", 39 | :short => "-s SCHEME", 40 | :long => "--scheme SCHEME", 41 | :default => "#{Socket.gethostname}.redis" 42 | 43 | def run 44 | redis = Redis.new(:host => config[:host], :port => config[:port]) 45 | 46 | redis.info.each do |k, v| 47 | next unless SKIP_KEYS_REGEX.map { |re| k.match(/#{re}/)}.compact.empty? 48 | 49 | # "db0"=>"keys=123,expires=12" 50 | if k =~ /^db/ 51 | keys, expires = v.split(',') 52 | keys.gsub!('keys=', '') 53 | expires.gsub!('expires=', '') 54 | 55 | output "#{config[:scheme]}.#{k}.keys", keys 56 | output "#{config[:scheme]}.#{k}.expires", expires 57 | else 58 | output "#{config[:scheme]}.#{k}", v 59 | end 60 | end 61 | 62 | ok 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /files/default/handlers/chef_node.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # This handler removes a Sensu client if its Chef node data 4 | # no longer exists. 5 | # 6 | # Requires the following Rubygems (`gem install $GEM`): 7 | # - sensu-plugin 8 | # - spice (~> 1.0.6) 9 | # - rest-client 10 | # 11 | # Requires a Sensu configuration snippet: 12 | # { 13 | # "chef": { 14 | # "server_url": "https://api.opscode.com:443/organizations/vulcan", 15 | # "client_name": "spock", 16 | # "client_key": "/path/to/spocks/key.pem" 17 | # } 18 | # } 19 | # 20 | # Best to use this handler with a filter: 21 | # { 22 | # "filters": { 23 | # "keepalives": { 24 | # "attributes": { 25 | # "check": { 26 | # "name": "keepalive" 27 | # } 28 | # } 29 | # } 30 | # }, 31 | # "handlers": { 32 | # "chef_node": { 33 | # "type": "pipe", 34 | # "command": "chef_node.rb", 35 | # "filter": "keepalives" 36 | # } 37 | # } 38 | # } 39 | # 40 | # Copyright 2013 Heavy Water Operations, LLC. 41 | # 42 | # Released under the same terms as Sensu (the MIT license); see 43 | # LICENSE for details. 44 | 45 | require "rubygems" 46 | require "sensu-handler" 47 | require "spice" 48 | require "rest_client" 49 | require "timeout" 50 | 51 | class ChefNode < Sensu::Handler 52 | def chef_node_exists? 53 | Spice.setup do |s| 54 | s.server_url = settings["chef"]["server_url"] 55 | s.client_name = settings["chef"]["client_name"] 56 | s.client_key = Spice.read_key_file(settings["chef"]["client_key"]) 57 | s.chef_version = settings["chef"]["version"] || "11.0.0" 58 | unless settings["chef"]["verify_ssl"].nil? 59 | s.connection_options = { 60 | :ssl => { 61 | :verify => settings["chef"]["verify_ssl"] 62 | } 63 | } 64 | end 65 | end 66 | begin 67 | timeout(8) do 68 | !!Spice.node(@event["client"]["name"]) 69 | end 70 | rescue Spice::Error::NotFound 71 | false 72 | rescue => error 73 | puts "Chef Node - Unexpected error: #{error.message}" 74 | true 75 | end 76 | end 77 | 78 | def delete_sensu_client! 79 | client_name = @event["client"]["name"] 80 | api_url = "http://" 81 | api_url << settings["api"]["host"] 82 | api_url << ":" 83 | api_url << settings["api"]["port"].to_s 84 | api_url << "/clients/" 85 | api_url << client_name 86 | begin 87 | timeout(8) do 88 | RestClient::Resource.new( 89 | api_url, 90 | :user => settings["api"]["user"], 91 | :password => settings["api"]["password"] 92 | ).delete 93 | end 94 | puts "Chef Node - Successfully deleted Sensu client: #{client_name}" 95 | rescue => error 96 | puts "Chef Node - Unexpected error: #{error.message}" 97 | end 98 | end 99 | 100 | def filter; end 101 | 102 | def handle 103 | unless chef_node_exists? 104 | delete_sensu_client! 105 | end 106 | end 107 | end 108 | -------------------------------------------------------------------------------- /files/default/plugins/check-haproxy.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # HAProxy Check 4 | # === 5 | # 6 | # Defaults to checking if ALL services in the given group are up; with 7 | # -1, checks if ANY service is up. with -A, checks all groups. 8 | # 9 | # Updated: To add -S to allow for different named sockets 10 | # 11 | # Copyright 2011 Sonian, Inc. 12 | # 13 | # Released under the same terms as Sensu (the MIT license); see LICENSE 14 | # for details. 15 | 16 | require "rubygems" if RUBY_VERSION < "1.9.0" 17 | require "sensu-plugin/check/cli" 18 | 19 | def silent_require(buggy_gem) 20 | dup_stderr = STDERR.dup 21 | STDERR.reopen("/dev/null") 22 | unless require(buggy_gem) 23 | echo "haproxy gem not installed" 24 | exit 1 25 | end 26 | STDERR.reopen(dup_stderr) 27 | end 28 | 29 | silent_require "haproxy" 30 | 31 | class CheckHAProxy < Sensu::Plugin::Check::CLI 32 | 33 | option :warn_percent, 34 | :short => "-w PERCENT", 35 | :boolean => true, 36 | :default => 50, 37 | :proc => proc {|a| a.to_i }, 38 | :description => "Warning Percent, default: 50" 39 | 40 | option :crit_percent, 41 | :short => "-c PERCENT", 42 | :boolean => true, 43 | :default => 25, 44 | :proc => proc {|a| a.to_i }, 45 | :description => "Critical Percent, default: 25" 46 | 47 | option :all_services, 48 | :short => "-A", 49 | :boolean => true, 50 | :description => "Check ALL Services, flag enables" 51 | 52 | option :missing_ok, 53 | :short => "-m", 54 | :boolean => true, 55 | :description => "Missing OK, flag enables" 56 | 57 | option :service, 58 | :short => "-s SVC", 59 | :description => "Service Name to Check" 60 | 61 | option :socket, 62 | :short => "-S SOCKET", 63 | :default => "/var/run/haproxy.sock", 64 | :description => "Path to HAProxy Socket, default: /var/run/haproxy.sock" 65 | 66 | def run 67 | if config[:service] or config[:all_services] 68 | services = get_services 69 | else 70 | unknown "No service specified" 71 | end 72 | 73 | if services.empty? 74 | message "No services matching /#{config[:service]}/" 75 | if config[:missing_ok] 76 | ok 77 | else 78 | warning 79 | end 80 | else 81 | percent_up = 100 * services.select {|svc| svc[:status] == "UP" }.size / services.size 82 | failed_names = services.reject {|svc| svc[:status] == "UP" }.map {|svc| svc[:svname] } 83 | services_down = failed_names.empty? ? "" : ", DOWN: #{failed_names.join(', ')}" 84 | message "UP: #{percent_up}% of #{services.size} /#{config[:service]}/ services" + services_down 85 | if percent_up < config[:crit_percent] 86 | critical 87 | elsif percent_up < config[:warn_percent] 88 | warning 89 | else 90 | ok 91 | end 92 | end 93 | end 94 | 95 | def get_services 96 | haproxy = HAProxy.read_stats(config[:socket]) 97 | if config[:all_services] 98 | haproxy.stats 99 | else 100 | haproxy.stats.select do |svc| 101 | svc[:pxname] =~ /#{config[:service]}/ 102 | end.reject do |svc| 103 | ["FRONTEND", "BACKEND"].include?(svc[:svname]) 104 | end 105 | end 106 | end 107 | 108 | end 109 | -------------------------------------------------------------------------------- /files/default/plugins/check-log.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Check Log Plugin 4 | # === 5 | # 6 | # This plugin checks a log file for a regular expression, skipping lines 7 | # that have already been read, like Nagios's check_log. However, instead 8 | # of making a backup copy of the whole log file (very slow with large 9 | # logs), it stores the number of bytes read, and seeks to that position 10 | # next time. 11 | # 12 | # Copyright 2011 Sonian, Inc 13 | # 14 | # Released under the same terms as Sensu (the MIT license); see LICENSE 15 | # for details. 16 | 17 | require 'rubygems' if RUBY_VERSION < '1.9.0' 18 | require 'sensu-plugin/check/cli' 19 | require 'fileutils' 20 | 21 | class CheckLog < Sensu::Plugin::Check::CLI 22 | 23 | BASE_DIR = '/var/cache/check-log' 24 | 25 | option :state_auto, 26 | :description => "Set state file dir automatically using name", 27 | :short => '-n NAME', 28 | :long => '--name NAME', 29 | :proc => proc {|arg| "#{BASE_DIR}/#{arg}" } 30 | 31 | option :state_dir, 32 | :description => "Dir to keep state files under", 33 | :short => '-s DIR', 34 | :long => '--state-dir DIR', 35 | :default => "#{BASE_DIR}/default" 36 | 37 | option :log_file, 38 | :description => "Path to log file", 39 | :short => '-f FILE', 40 | :long => '--log-file FILE' 41 | 42 | option :pattern, 43 | :description => "Pattern to search for", 44 | :short => '-q PAT', 45 | :long => '--pattern PAT' 46 | 47 | option :warn, 48 | :description => "Warning level if pattern has a group", 49 | :short => '-w N', 50 | :long => '--warn N', 51 | :proc => proc {|a| a.to_i } 52 | 53 | option :crit, 54 | :description => "Critical level if pattern has a group", 55 | :short => '-c N', 56 | :long => '--crit N', 57 | :proc => proc {|a| a.to_i } 58 | 59 | option :only_warn, 60 | :description => "Warn instead of critical on match", 61 | :short => '-o', 62 | :long => '--warn-only', 63 | :boolean => true 64 | 65 | def run 66 | unknown "No log file specified" unless config[:log_file] 67 | unknown "No pattern specified" unless config[:pattern] 68 | begin 69 | open_log 70 | rescue => e 71 | unknown "Could not open log file: #{e}" 72 | end 73 | n_warns, n_crits = search_log 74 | message "#{n_warns} warnings, #{n_crits} criticals" 75 | if n_crits > 0 76 | critical 77 | elsif n_warns > 0 78 | warning 79 | else 80 | ok 81 | end 82 | end 83 | 84 | def open_log 85 | state_dir = config[:state_auto] || config[:state_dir] 86 | @log = File.open(config[:log_file]) 87 | @state_file = File.join(state_dir, File.expand_path(config[:log_file])) 88 | @bytes_to_skip = begin 89 | File.open(@state_file) do |file| 90 | file.readline.to_i 91 | end 92 | rescue 93 | 0 94 | end 95 | end 96 | 97 | def search_log 98 | log_file_size = @log.stat.size 99 | if log_file_size < @bytes_to_skip 100 | @bytes_to_skip = 0 101 | end 102 | bytes_read = 0 103 | n_warns = 0 104 | n_crits = 0 105 | if @bytes_to_skip > 0 106 | @log.seek(@bytes_to_skip, File::SEEK_SET) 107 | end 108 | @log.each_line do |line| 109 | bytes_read += line.size 110 | if m = line.match(config[:pattern]) 111 | if m[1] 112 | if config[:crit] && m[1].to_i > config[:crit] 113 | n_crits += 1 114 | elsif config[:warn] && m[1].to_i > config[:warn] 115 | n_warns += 1 116 | end 117 | else 118 | if config[:only_warn] 119 | n_warns += 1 120 | else 121 | n_crits += 1 122 | end 123 | end 124 | end 125 | end 126 | FileUtils.mkdir_p(File.dirname(@state_file)) 127 | File.open(@state_file, 'w') do |file| 128 | file.write(@bytes_to_skip + bytes_read) 129 | end 130 | [n_warns, n_crits] 131 | end 132 | 133 | end 134 | -------------------------------------------------------------------------------- /files/default/plugins/check-http.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Check HTTP 4 | # === 5 | # 6 | # Takes either a URL or a combination of host/path/port/ssl, and checks for 7 | # a 200 response (that matches a pattern, if given). Can use client certs. 8 | # 9 | # Copyright 2011 Sonian, Inc 10 | # Updated by Lewis Preson 2012 to accept basic auth credentials 11 | # Updated by SweetSpot 2012 to require specified redirect 12 | # 13 | # Released under the same terms as Sensu (the MIT license); see LICENSE 14 | # for details. 15 | 16 | require 'rubygems' if RUBY_VERSION < '1.9.0' 17 | require 'sensu-plugin/check/cli' 18 | require 'net/http' 19 | require 'net/https' 20 | 21 | class CheckHTTP < Sensu::Plugin::Check::CLI 22 | 23 | option :url, 24 | :short => '-u URL', 25 | :long => '--url URL', 26 | :description => 'A URL to connect to' 27 | 28 | option :host, 29 | :short => '-h HOST', 30 | :long => '--hostname HOSTNAME', 31 | :description => 'A HOSTNAME to connect to' 32 | 33 | option :path, 34 | :short => '-p PATH', 35 | :long => '--path PATH', 36 | :description => 'Check a PATH' 37 | 38 | option :port, 39 | :short => '-P PORT', 40 | :long => '--port PORT', 41 | :proc => proc { |a| a.to_i }, 42 | :description => 'Select another port', 43 | :default => 80 44 | 45 | option :header, 46 | :short => '-H HEADER', 47 | :long => '--header HEADER', 48 | :description => 'Check for a HEADER' 49 | 50 | option :ssl, 51 | :short => '-s', 52 | :boolean => true, 53 | :description => 'Enabling SSL connections', 54 | :default => false 55 | 56 | option :insecure, 57 | :short => '-k', 58 | :boolean => true, 59 | :description => 'Enabling insecure connections', 60 | :default => false 61 | 62 | option :user, 63 | :short => '-U', 64 | :long => '--username USER', 65 | :description => 'A username to connect as' 66 | 67 | option :password, 68 | :short => '-a', 69 | :long => '--password PASS', 70 | :description => 'A password to use for the username' 71 | 72 | option :cert, 73 | :short => '-c FILE', 74 | :long => '--cert FILE', 75 | :description => 'Cert to use' 76 | 77 | option :cacert, 78 | :short => '-C FILE', 79 | :long => '--cacert FILE', 80 | :description => 'A CA Cert to use' 81 | 82 | option :pattern, 83 | :short => '-q PAT', 84 | :long => '--query PAT', 85 | :description => 'Query for a specific pattern' 86 | 87 | option :timeout, 88 | :short => '-t SECS', 89 | :long => '--timeout SECS', 90 | :proc => proc { |a| a.to_i }, 91 | :description => 'Set the timeout', 92 | :default => 15 93 | 94 | option :redirectok, 95 | :short => '-r', 96 | :boolean => true, 97 | :description => 'Check if a redirect is ok', 98 | :default => false 99 | 100 | option :redirectto, 101 | :short => '-R URL', 102 | :long => '--redirect-to URL', 103 | :description => 'Redirect to another page' 104 | 105 | def run 106 | if config[:url] 107 | uri = URI.parse(config[:url]) 108 | config[:host] = uri.host 109 | config[:path] = uri.path 110 | config[:port] = uri.port 111 | config[:ssl] = uri.scheme == 'https' 112 | else 113 | unless config[:host] and config[:path] 114 | unknown 'No URL specified' 115 | end 116 | config[:port] ||= config[:ssl] ? 443 : 80 117 | end 118 | 119 | begin 120 | timeout(config[:timeout]) do 121 | get_resource 122 | end 123 | rescue Timeout::Error 124 | critical "Request timed out" 125 | rescue => e 126 | critical "Request error: #{e.message}" 127 | end 128 | end 129 | 130 | def get_resource 131 | http = Net::HTTP.new(config[:host], config[:port]) 132 | 133 | if config[:ssl] 134 | http.use_ssl = true 135 | if config[:cert] 136 | cert_data = File.read(config[:cert]) 137 | http.cert = OpenSSL::X509::Certificate.new(cert_data) 138 | http.key = OpenSSL::PKey::RSA.new(cert_data, nil) 139 | end 140 | if config[:cacert] 141 | http.ca_file = config[:cacert] 142 | end 143 | if config[:insecure] 144 | http.verify_mode = OpenSSL::SSL::VERIFY_NONE 145 | end 146 | end 147 | 148 | req = Net::HTTP::Get.new(config[:path]) 149 | if (config[:user] != nil and config[:password] != nil) 150 | req.basic_auth config[:user], config[:password] 151 | end 152 | if config[:header] 153 | header, value = config[:header].split(':', 2) 154 | req[header] = value.strip 155 | end 156 | res = http.request(req) 157 | 158 | case res.code 159 | when /^2/ 160 | if config[:redirectto] 161 | critical "Expected redirect to #{config[:redirectto]} but got #{res.code}" 162 | elsif config[:pattern] 163 | if res.body =~ /#{config[:pattern]}/ 164 | ok "#{res.code}, found /#{config[:pattern]}/ in #{res.body.size} bytes" 165 | else 166 | critical "#{res.code}, did not find /#{config[:pattern]}/ in #{res.body.size} bytes: #{res.body[0...200]}..." 167 | end 168 | else 169 | ok "#{res.code}, #{res.body.size} bytes" 170 | end 171 | when /^3/ 172 | if config[:redirectok] || config[:redirectto] 173 | if config[:redirectok] 174 | ok "#{res.code}, #{res.body.size} bytes" 175 | elsif config[:redirectto] 176 | if config[:redirectto] == res['Location'] 177 | ok "#{res.code} found redirect to #{res['Location']}" 178 | else 179 | critical "Expected redirect to #{config[:redirectto]} instead redirected to #{res['Location']}" 180 | end 181 | end 182 | else 183 | warning res.code 184 | end 185 | when /^4/, /^5/ 186 | critical res.code 187 | else 188 | warning res.code 189 | end 190 | end 191 | end 192 | -------------------------------------------------------------------------------- /files/default/plugins/rabbitmq-overview-metrics.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # 4 | # RabbitMQ Overview Metrics 5 | # === 6 | # 7 | # Dependencies 8 | # ----------- 9 | # - RabbitMQ `rabbitmq_management` plugin 10 | # - Ruby gem `carrot-top` 11 | # 12 | # Overview stats 13 | # --------------- 14 | # RabbitMQ 'overview' stats are similar to what is shown on the main page 15 | # of the rabbitmq_management web UI. Example: 16 | # 17 | # $ rabbitmq-queue-metrics.rb 18 | # host.rabbitmq.queue_totals.messages.count 0 1344186404 19 | # host.rabbitmq.queue_totals.messages.rate 0.0 1344186404 20 | # host.rabbitmq.queue_totals.messages_unacknowledged.count 0 1344186404 21 | # host.rabbitmq.queue_totals.messages_unacknowledged.rate 0.0 1344186404 22 | # host.rabbitmq.queue_totals.messages_ready.count 0 1344186404 23 | # host.rabbitmq.queue_totals.messages_ready.rate 0.0 1344186404 24 | # host.rabbitmq.message_stats.publish.count 4605755 1344186404 25 | # host.rabbitmq.message_stats.publish.rate 17.4130186829638 1344186404 26 | # host.rabbitmq.message_stats.deliver_no_ack.count 6661111 1344186404 27 | # host.rabbitmq.message_stats.deliver_no_ack.rate 24.6867565643405 1344186404 28 | # host.rabbitmq.message_stats.deliver_get.count 6661111 1344186404 29 | # host.rabbitmq.message_stats.deliver_get.rate 24.6867565643405 1344186404# 30 | # 31 | # Copyright 2012 Joe Miller - https://github.com/joemiller 32 | # 33 | # Released under the same terms as Sensu (the MIT license); see LICENSE 34 | # for details. 35 | 36 | require 'rubygems' if RUBY_VERSION < '1.9.0' 37 | require 'sensu-plugin/metric/cli' 38 | require 'socket' 39 | require 'carrot-top' 40 | 41 | class RabbitMQMetrics < Sensu::Plugin::Metric::CLI::Graphite 42 | 43 | option :host, 44 | :description => "RabbitMQ management API host", 45 | :long => "--host HOST", 46 | :default => "localhost" 47 | 48 | option :port, 49 | :description => "RabbitMQ management API port", 50 | :long => "--port PORT", 51 | :proc => proc {|p| p.to_i}, 52 | :default => 15672 53 | 54 | option :user, 55 | :description => "RabbitMQ management API user", 56 | :long => "--user USER", 57 | :default => "guest" 58 | 59 | option :password, 60 | :description => "RabbitMQ management API password", 61 | :long => "--password PASSWORD", 62 | :default => "guest" 63 | 64 | option :scheme, 65 | :description => "Metric naming scheme, text to prepend to $queue_name.$metric", 66 | :long => "--scheme SCHEME", 67 | :default => "#{Socket.gethostname}.rabbitmq" 68 | 69 | option :ssl, 70 | :description => "Enable SSL for connection to the API", 71 | :long => "--ssl", 72 | :boolean => true, 73 | :default => false 74 | 75 | def get_rabbitmq_info 76 | begin 77 | rabbitmq_info = CarrotTop.new( 78 | :host => config[:host], 79 | :port => config[:port], 80 | :user => config[:user], 81 | :password => config[:password], 82 | :ssl => config[:ssl] 83 | ) 84 | rescue 85 | warning "could not get rabbitmq info" 86 | end 87 | rabbitmq_info 88 | end 89 | 90 | def run 91 | timestamp = Time.now.to_i 92 | 93 | rabbitmq = get_rabbitmq_info 94 | overview = rabbitmq.overview 95 | 96 | # overview['queue_totals']['messages'] 97 | if overview.has_key?('queue_totals') && !overview['queue_totals'].empty? 98 | output "#{config[:scheme]}.queue_totals.messages.count", overview['queue_totals']['messages'], timestamp 99 | output "#{config[:scheme]}.queue_totals.messages.rate", overview['queue_totals']['messages_details']['rate'], timestamp 100 | 101 | # overview['queue_totals']['messages_unacknowledged'] 102 | output "#{config[:scheme]}.queue_totals.messages_unacknowledged.count", overview['queue_totals']['messages_unacknowledged'], timestamp 103 | output "#{config[:scheme]}.queue_totals.messages_unacknowledged.rate", overview['queue_totals']['messages_unacknowledged_details']['rate'], timestamp 104 | 105 | # overview['queue_totals']['messages_ready'] 106 | output "#{config[:scheme]}.queue_totals.messages_ready.count", overview['queue_totals']['messages_ready'], timestamp 107 | output "#{config[:scheme]}.queue_totals.messages_ready.rate", overview['queue_totals']['messages_ready_details']['rate'], timestamp 108 | end 109 | 110 | if overview.has_key?('message_stats') && !overview['message_stats'].empty? 111 | # overview['message_stats']['publish'] 112 | if overview['message_stats'].include?('publish') 113 | output "#{config[:scheme]}.message_stats.publish.count", overview['message_stats']['publish'], timestamp 114 | end 115 | if overview['message_stats'].include?('publish_details') && 116 | overview['message_stats']['publish_details'].include?('rate') 117 | output "#{config[:scheme]}.message_stats.publish.rate", overview['message_stats']['publish_details']['rate'], timestamp 118 | end 119 | 120 | # overview['message_stats']['deliver_no_ack'] 121 | if overview['message_stats'].include?('deliver_no_ack') 122 | output "#{config[:scheme]}.message_stats.deliver_no_ack.count", overview['message_stats']['deliver_no_ack'], timestamp 123 | end 124 | if overview['message_stats'].include?('deliver_no_ack_details') && 125 | overview['message_stats']['deliver_no_ack_details'].include?('rate') 126 | output "#{config[:scheme]}.message_stats.deliver_no_ack.rate", overview['message_stats']['deliver_no_ack_details']['rate'], timestamp 127 | end 128 | 129 | # overview['message_stats']['deliver_get'] 130 | if overview['message_stats'].include?('deliver_get') 131 | output "#{config[:scheme]}.message_stats.deliver_get.count", overview['message_stats']['deliver_get'], timestamp 132 | end 133 | if overview['message_stats'].include?('deliver_get_details') && 134 | overview['message_stats']['deliver_get_details'].include?('rate') 135 | output "#{config[:scheme]}.message_stats.deliver_get.rate", overview['message_stats']['deliver_get_details']['rate'], timestamp 136 | end 137 | end 138 | ok 139 | end 140 | 141 | end 142 | -------------------------------------------------------------------------------- /files/default/extensions/statsd.rb: -------------------------------------------------------------------------------- 1 | # Statsd 2 | # === 3 | # 4 | # Runs a Statsd socket/implementation within the Sensu Ruby VM. 5 | # Expects a "graphite" handler on the Sensu server, eg: 6 | # 7 | # "graphite": { 8 | # "type": "tcp", 9 | # "socket": { 10 | # "host": "graphite.hw-ops.com", 11 | # "port": 2003 12 | # }, 13 | # "mutator": "only_check_output" 14 | # } 15 | # 16 | # Copyright 2014 Heavy Water Operations, LLC. 17 | # 18 | # Released under the same terms as Sensu (the MIT license); see LICENSE 19 | # for details. 20 | 21 | module Sensu 22 | module Extension 23 | class SimpleSocket < EM::Connection 24 | attr_accessor :data 25 | 26 | def receive_data(data) 27 | @data << data 28 | end 29 | end 30 | 31 | class Statsd < Check 32 | def name 33 | 'statsd' 34 | end 35 | 36 | def description 37 | 'a statsd implementation' 38 | end 39 | 40 | def options 41 | return @options if @options 42 | @options = { 43 | :bind => '127.0.0.1', 44 | :port => 8125, 45 | :flush_interval => 10, 46 | :send_interval => 30, 47 | :percentile => 90, 48 | :add_client_prefix => true, 49 | :path_prefix => 'statsd', 50 | :handler => 'graphite' 51 | } 52 | if @settings[:statsd].is_a?(Hash) 53 | @options.merge!(@settings[:statsd]) 54 | end 55 | @options 56 | end 57 | 58 | def definition 59 | { 60 | :type => 'metric', 61 | :name => name, 62 | :interval => options[:send_interval], 63 | :standalone => true, 64 | :handler => options[:handler] 65 | } 66 | end 67 | 68 | def post_init 69 | @flush_timers = [] 70 | @data = EM::Queue.new 71 | @gauges = Hash.new { |h, k| h[k] = 0 } 72 | @counters = Hash.new { |h, k| h[k] = 0 } 73 | @timers = Hash.new { |h, k| h[k] = [] } 74 | @metrics = [] 75 | setup_flush_timers 76 | setup_parser 77 | setup_statsd_socket 78 | end 79 | 80 | def run 81 | output = '' 82 | output << @metrics.join("\n") + "\n" unless @metrics.empty? 83 | @logger.info('statsd collected metrics', { 84 | :count => @metrics.count 85 | }) 86 | @metrics = [] 87 | yield output, 0 88 | end 89 | 90 | private 91 | 92 | def add_metric(*args) 93 | value = args.pop 94 | path = [] 95 | if options[:add_client_prefix] 96 | path << @settings[:client][:name] 97 | end 98 | path << options[:path_prefix] 99 | path = (path + args).join('.') 100 | if path !~ /^[A-Za-z0-9\._-]*$/ 101 | @logger.info('invalid statsd metric', { 102 | :reason => 'metric path must only consist of alpha-numeric characters, periods, underscores, and dashes', 103 | :path => path, 104 | :value => value 105 | }) 106 | else 107 | @logger.debug('adding statsd metric', { 108 | :path => path, 109 | :value => value 110 | }) 111 | @metrics << [path, value, Time.now.to_i].join(' ') 112 | end 113 | end 114 | 115 | def flush! 116 | @gauges.each do |name, value| 117 | add_metric('guages', name, value) 118 | end 119 | @gauges.clear 120 | @counters.each do |name, value| 121 | add_metric('counters', name, value) 122 | end 123 | @counters.clear 124 | @timers.each do |name, values| 125 | values.sort! 126 | length = values.length 127 | min = values.first 128 | max = values.last 129 | mean = min 130 | max_at_threshold = min 131 | percentile = options[:percentile] 132 | if length > 1 133 | threshold_index = ((100 - percentile) / 100.0) * length 134 | threshold_count = length - threshold_index.round 135 | valid_values = values.slice(0, threshold_count) 136 | max_at_threshold = valid_values[-1] 137 | sum = 0 138 | valid_values.each { |v| sum += v } 139 | mean = sum / valid_values.length 140 | end 141 | add_metric('timers', name, 'lower', min) 142 | add_metric('timers', name, 'mean', mean) 143 | add_metric('timers', name, 'upper', max) 144 | add_metric('timers', name, 'upper_' + percentile.to_s, max_at_threshold) 145 | end 146 | @timers.clear 147 | @logger.debug('flushed statsd metrics') 148 | end 149 | 150 | def setup_flush_timers 151 | @flush_timers << EM::PeriodicTimer.new(options[:flush_interval]) do 152 | flush! 153 | end 154 | end 155 | 156 | def setup_parser 157 | parser = Proc.new do |data| 158 | begin 159 | nv, type = data.strip.split('|') 160 | name, value = nv.split(':') 161 | case type 162 | when 'g' 163 | @gauges[name] = Float(value) 164 | when /^c/, 'm' 165 | _, raw_sample = type.split('@') 166 | sample = (raw_sample ? Float(raw_sample) : 1) 167 | @counters[name] += Integer(value) * (1 / sample) 168 | when 'ms', 'h' 169 | @timers[name] << Float(value) 170 | end 171 | rescue => error 172 | @logger.error('statsd parser error', { 173 | :error => error.to_s 174 | }) 175 | end 176 | EM.next_tick do 177 | @data.pop(&parser) 178 | end 179 | end 180 | @data.pop(&parser) 181 | end 182 | 183 | def setup_statsd_socket 184 | @logger.debug('binding statsd tcp and udp sockets', { 185 | :options => options 186 | }) 187 | bind = options[:bind] 188 | port = options[:port] 189 | EM.start_server(bind, port, SimpleSocket) do |socket| 190 | socket.data = @data 191 | end 192 | EM.open_datagram_socket(bind, port, SimpleSocket) do |socket| 193 | socket.data = @data 194 | end 195 | end 196 | end 197 | end 198 | end 199 | -------------------------------------------------------------------------------- /test/integration/data_bags/sensu/ssl.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "ssl", 3 | "server": { 4 | "key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEAv38c0+6IqGKdVu17onq7qC+4PrY9U8xo0XHb1NMeDuKXgGDD\nJI/BgidimA+ckLs3A/mMHToAVyYd+XctdgGcngaZsmex3mXZYikloOPPeZSDCh5N\n+JNB3/a58BDzHH+FEXixnmBd2Maus7yI/n8jMmi+/b/PkFGsr4bzFZW9xw4qKh3+\nLRtwyJW6ge298TRN2Fn+jOSBx4XnpDvMPdUIO4cK+tRostM0AT/oTZjBBxkYtZkp\ndJ6wXLHOwWxr8K7CUPsMab/tM+gD7FNg/Kdhyh/MbQ1fe43ARm+eAS9hi3dz1Zq1\n8F5UNhjaP/gDIir/giHmGNUhn1GMRuooQkZvzQIDAQABAoIBAGrhJVPcFIX21F5K\nGfErk0VhrPHDgSMeOXp8k/eLBYHFUk2itAfZ/GUKWmsIe5gEBcbAJ0XAs6PwcFZm\nAAeXCrTgdjQu0AWSzuuMj6XmwEvZafHhzB7rAZ9g85Jbw3Sqb7i6plgA7SAqdCN5\nHPT+PMTL1Z2GyvB/xWQKa+4mL0/DThsjOjeIDV+kkB2mlZO7FwQoB3Pwdv1DTlKP\nmP/l7pB+GxfkHEJcULXTjw9MstfNgrxVd5Xlgs8dXtsYymagWHtuKQHJ3jhptZwk\nnB/IqMWIr1oPD+vWfJLC/Jk0tG3ygHMJpMhm3pwvKt5Y23Ackx5MCRP7bjk1LvpH\nItD/vcECgYEA8MQrWcHDWdpxUSTsMFhUaSsmEc6JrquGReu2kuLf4GlvS78O1B8u\np4YhZyfHJJhHvH/wjocG8lrHpQmE9+O9gaOJbrwkaLapQ/KoRnN82DSj7FK/4J44\nxhcE8Qbuy3/FKjvs5eE1WVkFUjC3hi3xo+t0NfXIKxOnsRmyaQ6HedkCgYEAy5zk\ngmLYgpndI1itCq0uJeHTeIljjXW01QIs/KptyFrX73RwbsCSF7Ls3PquI/P5MXnz\nb7G+7iP0kDRmdL2SSTQb7yv/p82MlvpIYKV92Llu6FP7w68U1oLJ7tzDs/7pA8/j\n+/gREK2j76iKT/WU67NZ6t7IBiX2QM1ZIyEVWRUCgYEAzA0FzIlKPgHDXgkT35lM\n2OEH37mSuKWIJAQ0dOKw6KmS+LhRIffXe4VTE/EDdwFUcu5fevv3KDlF0Jpzxf+O\noatvE5mCpfiBFiQ295ZzLW3Xq7cqBX0zErd5qai9g1yjRjsJeH5Yz4OB3fmJ4L9Z\ndxpbIExQvClpStBDg4Qbz2kCgYEAsqV0hMXvSx9uvB2Ire5mqAwse/ynYS1ePOpM\nyF9MNVcnUMxr/XBufeaaFgTIF3lvMa7bHSvp5o2bfY7MOi8+E4Zp3/5NsFATwz/T\nkyCAe0+vdJ6DLh0GfnOAc1g1SU8l6RdU4/WyG10Oki7hM+i4Adl50i4nd+WE2zSQ\nudbEIUUCgYEAlUfSUGPHv+rS6cXm0mIYaquRBMu64ed+LUSHL/QWrUt77lDenAH+\n/FUp1+vaozA9hKjWLNP1RnvjyyDaaQqkp/5y2BhSHNUbEfwG6J5YyyPo46RXXZEh\n1qe9Z9w8UPuk22aOSL6fd7ayJUty8uShr3XrVDh1YaccB0gbhnwXdbA=\n-----END RSA PRIVATE KEY-----\n", 5 | "cert": "-----BEGIN CERTIFICATE-----\nMIIC3TCCAcWgAwIBAgIBATANBgkqhkiG9w0BAQUFADASMRAwDgYDVQQDEwdTZW5z\ndUNBMB4XDTEyMTIyMjA4NDAwMFoXDTE3MTIyMTA4NDAwMFowITEOMAwGA1UEAwwF\nc2Vuc3UxDzANBgNVBAoMBnNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBAL9/HNPuiKhinVbte6J6u6gvuD62PVPMaNFx29TTHg7il4BgwySPwYIn\nYpgPnJC7NwP5jB06AFcmHfl3LXYBnJ4GmbJnsd5l2WIpJaDjz3mUgwoeTfiTQd/2\nufAQ8xx/hRF4sZ5gXdjGrrO8iP5/IzJovv2/z5BRrK+G8xWVvccOKiod/i0bcMiV\nuoHtvfE0TdhZ/ozkgceF56Q7zD3VCDuHCvrUaLLTNAE/6E2YwQcZGLWZKXSesFyx\nzsFsa/CuwlD7DGm/7TPoA+xTYPynYcofzG0NX3uNwEZvngEvYYt3c9WatfBeVDYY\n2j/4AyIq/4Ih5hjVIZ9RjEbqKEJGb80CAwEAAaMvMC0wCQYDVR0TBAIwADALBgNV\nHQ8EBAMCBSAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDQYJKoZIhvcNAQEFBQADggEB\nALJUshCBu+BXzOmmAtzH02MdqaLXgZFcccHSwsOgu/h/AeAd+DLCWvuq5VxXN3bF\n65TmCL6vfqdfIp+cK6v3iXjGJGNdhaovGVucKNjTxboqww2wLLajIdVwVlfp795s\nu8FCDQlrcTLWfOE5b80HXROzdF0RrnMTBjqFGICp0Y+X119BLLlTszvqiWos2RcZ\n6bQzYAFJksSt7Mmx7MFeGqOs9U0dBMipflMuXxK2BDGrk8ZPq6ZLASyIgiTSAOJv\nsMFQF7c+Ul+7fQXVerg7O0LcPmmUIFPr4FJfNWjtAk7bJM5lepHE4QD/U2LhdtWi\n0GgzLe7v/glMzP6ofPelGVI=\n-----END CERTIFICATE-----\n", 6 | "cacert": "-----BEGIN CERTIFICATE-----\nMIICxDCCAaygAwIBAgIJAM2e0X9c/B+9MA0GCSqGSIb3DQEBBQUAMBIxEDAOBgNV\nBAMTB1NlbnN1Q0EwHhcNMTIxMjIyMDg0MDAwWhcNMTcxMjIxMDg0MDAwWjASMRAw\nDgYDVQQDEwdTZW5zdUNBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\nys/vfnDDerLEE1Qq+hREOjzWYFioiS8TsMWL7iJc0p3XouRqYDlGYW/EkRG3eEvr\nOXRhOgk94c31n3YpoPwTqRAnmCHnFAzHSYa7hr0z2b2KIwENdpsoQqiVBPCwc+TH\nOLlTcnwKebsS1sFW4R3xv2i7D987UyedAcMk45hQsAgr8M3/AD2FGiaslCxt+Tx4\nAvAoX/Vr9IKeCmE+WvkxlaEodhtHHiVWyiZAwMF72WwIONNoOnskkAf18O/dr+pF\nt588q9G6gkrl7dxsutlIzr0qGy7gPEB9xaGR8sGTkuDB0c5RHaAJcyzBMk8b6owD\nxWvhDDm4ikxUO7XFGjhh9QIDAQABox0wGzAMBgNVHRMEBTADAQH/MAsGA1UdDwQE\nAwIBBjANBgkqhkiG9w0BAQUFAAOCAQEAbc44LDHTm0p4KyKbtw3UX8389hiRZTsW\nN9z7gF5RbPZDc6kJGawGIag8wPfzsfQcNaXa75c/4MQJIdc0r4Qf0j+7HY1lORCv\nqyYUmR5opFclJppajIS011bfISNpfiflU+v7AP30AX/Mb4FyRyz4azASP2xsI1zn\nm50MSC9uxa8RynHwrT9kBuYZEKjsef7e1Up+u0LWEziJ2ifmpr6efX/QQ3xFDr2m\nh+dPZiljq04oHIsvtynrOSqcMcTtZb9ZWPCpppHZRrvHQZ2X8yNwwRH9g1l/V+TG\nh7Lp9EZwxHo43SigZQQd5wvlZoqF/CDGv0YH63Mq1/mpU/a9Pb2BrA==\n-----END CERTIFICATE-----\n" 7 | }, 8 | "client": { 9 | "key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA51fcWpL4flJrV3Ej3LbOV6akgx6r80LLgzn9LrjwUdf7/od5\nvOuaRYgytSFiFHfRO2BkYf9oCZpDLIRlyo4HGehjsCqsmuOmalIoeU+8P7CypQe/\nKgljOPkOLbZBZSm627WVV8bYXPXELiSHPd63dC33VJlIaN9HlMQu9JyUiHf8wmt4\nVd/fOG70mdQURU9bZ05xK7YrsedASTbD4qpQR6yWM+qxL8wdQhpR/UC53r6uDz1y\nKQRWb96JAwwImolckpohGr4zUAUiq+cB5syZkkA1ziMeCyaguQORrcfPJgNQ/MLh\n+f1WbdiY7mXYuSb7pqpoLG/90a7BkwysprC8LwIDAQABAoIBAEVo4A0w0WudMOWQ\nZZEgK/KyZeONPUwEO2lI6cSt6jS0F8A7R3Q+QBAimAwuUu4PMPCesB+S8NUIuqTN\nqHjIeir0xYwikZz7Nn/p3IjhOnrlegP3ugxTurhqSS2/5AkFJzIqXf9zApGJj23E\nuer/Oo2aCP4R617DtwdUgT4VCJ+UxkfD6+Fa5SowdCETD8V9xzpDssnx5jeYsUAf\nCKnoaIO0xbIA01xmFxnmAK2VS+lS0Z7LOtxkz5awla/rbGKUrMsDBUQeCa+ufRkX\nLf3jSdcB52klEcIh8R40YQXPWsTsm8N1zSoMVwKLMazV1+U9VFruDkNL2avtCstP\nObMcd3kCgYEA/zjvoeFrxqO7SFHf9NxpywGY/ZERlV7bxWLxWT6sCLsupUoEHAWQ\nOGMKyYIbPx0W55wU3VBXotJ3hTrA6wADbhACP1bjsqK4YJ/UfK+8thgLlB3119ZN\nE4vZ7Q3T+L22ofcauqlm52CNRMcI3n8m7t8Pg45LmyipOzLZmyiCbmMCgYEA6AxM\nwFF/2ILsHu+TWgdf29S9LzR1OHlqBAj2zvFQfck5PuBqc4HIjIJL2Co+jGPAB/E9\neBDA2xiaPbmegDMEskyyDBbZHLRuoyUG4UYzMApcBdUlSBd0ueVktYIsGYgdLhcf\nGn4fcu7nhPMzaPs3w94pizjc6NCXWHcCWYZULsUCgYAtpDj2ciGb/FvZqwqp92DT\n9Q0vC2phk/0pZ3BRzWtmhFLrLDlf6X7JFq0vLB1DRCh8cuUoTt3dOFY3dTJa1D3I\nyQQd36QIpiDi5rJROfw9dD9d1Z3JY4GVJUVrpzVpHkQy4sXc676DUjjxO8bSQ1rr\nDOjxVdkffj4FYCqaHLx2kQKBgQDien6+hczdqIqEdxe+Gsx4Zl5vICf4gqfyYD1j\nJSuGWUtCDw5WxijS2qjjxfdeVk0qc02nTdKtBIomDHCidyjyTODJ49LrrT7+hDj+\nzeFP1gyStMG6ZwpKFZYBgZdBJBQ20+JLnSVcq8fTfvqCUA7RYJfzhgA2SsAsW5sz\nfWPUcQKBgQDLbz1Hmv9oN8NMs7ESgsGmMzhr4e0GQyGJdEx4LtRPRVIL1Lk9id9E\nqoOE04ZzPzmbwLMPsAp+58EifnadavbEsw0eEIacn1cCdMNez26wruwwuvPrRgDO\nJQMx7y3tpQetiO7gSdLWpmz3YLTKfNbcbI2WhNNYHrZ6aSV9WdQ8Fg==\n-----END RSA PRIVATE KEY-----\n", 10 | "cert": "-----BEGIN CERTIFICATE-----\nMIIC3TCCAcWgAwIBAgIBAjANBgkqhkiG9w0BAQUFADASMRAwDgYDVQQDEwdTZW5z\ndUNBMB4XDTEyMTIyMjA4NDAwMFoXDTE3MTIyMTA4NDAwMFowITEOMAwGA1UEAwwF\nc2Vuc3UxDzANBgNVBAoMBmNsaWVudDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBAOdX3FqS+H5Sa1dxI9y2zlempIMeq/NCy4M5/S648FHX+/6HebzrmkWI\nMrUhYhR30TtgZGH/aAmaQyyEZcqOBxnoY7AqrJrjpmpSKHlPvD+wsqUHvyoJYzj5\nDi22QWUputu1lVfG2Fz1xC4khz3et3Qt91SZSGjfR5TELvSclIh3/MJreFXf3zhu\n9JnUFEVPW2dOcSu2K7HnQEk2w+KqUEesljPqsS/MHUIaUf1Aud6+rg89cikEVm/e\niQMMCJqJXJKaIRq+M1AFIqvnAebMmZJANc4jHgsmoLkDka3HzyYDUPzC4fn9Vm3Y\nmO5l2Lkm+6aqaCxv/dGuwZMMrKawvC8CAwEAAaMvMC0wCQYDVR0TBAIwADALBgNV\nHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYBBQUHAwIwDQYJKoZIhvcNAQEFBQADggEB\nAKKE8W5//wwJ7xRpCRce25nrrCh6hSAxf9dEzheJCkcZFycWHN8ry3tL0nvhdvTc\nGkzSDx4jG8ZuuaYp8jxdQRHBHudr1sPARBsC9HOd4fnhpP8qb3VB3vJMFCdKeiEZ\ndEaDUVVHqg4yY4ekjh7Yqu4bUu6mqOicYWbwp+UX8OyYMN3Z7NPD7vD51BCZ9+cv\nOIQKEX9rnIv/xW4OHGMIGG88m9gUny5T/ste9dJ56xsLwAhFXFDZOH8CiTFQdFRv\nHnaPzQyhG5+bLiBnkXKtXxxCEwYbbOweNFL2DT4i21+yY2Q69nTV3gJnLPwQuThb\nHNHH4eUraBt6sAcu+vWo6qU=\n-----END CERTIFICATE-----\n" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /files/default/extensions/system_profile.rb: -------------------------------------------------------------------------------- 1 | # System Profile (metrics) 2 | # === 3 | # 4 | # Collects a variety of system metrics every 10 seconds (by default). 5 | # Expects a "graphite" handler on the Sensu server, eg: 6 | # 7 | # "graphite": { 8 | # "type": "tcp", 9 | # "socket": { 10 | # "host": "graphite.hw-ops.com", 11 | # "port": 2003 12 | # }, 13 | # "mutator": "only_check_output" 14 | # } 15 | # 16 | # Copyright 2014 Heavy Water Operations, LLC. 17 | # 18 | # Released under the same terms as Sensu (the MIT license); see LICENSE 19 | # for details. 20 | 21 | module Sensu 22 | module Extension 23 | class SystemProfile < Check 24 | def name 25 | 'system_profile' 26 | end 27 | 28 | def description 29 | 'collects system metrics, using the graphite plain-text format' 30 | end 31 | 32 | def definition 33 | { 34 | :type => 'metric', 35 | :name => name, 36 | :interval => options[:interval], 37 | :standalone => true, 38 | :handler => options[:handler] 39 | } 40 | end 41 | 42 | def post_init 43 | @metrics = [] 44 | end 45 | 46 | def run 47 | proc_stat_metrics do 48 | proc_loadavg_metrics do 49 | proc_net_dev_metrics do 50 | proc_meminfo_metrics do 51 | yield flush_metrics, 0 52 | end 53 | end 54 | end 55 | end 56 | end 57 | 58 | private 59 | 60 | def options 61 | return @options if @options 62 | @options = { 63 | :interval => 10, 64 | :handler => 'graphite', 65 | :add_client_prefix => true, 66 | :path_prefix => 'system' 67 | } 68 | if settings[:system_profile].is_a?(Hash) 69 | @options.merge!(settings[:system_profile]) 70 | end 71 | @options 72 | end 73 | 74 | def flush_metrics 75 | metrics = @metrics.join("\n") + "\n" 76 | @metrics = [] 77 | metrics 78 | end 79 | 80 | def add_metric(*args) 81 | value = args.pop 82 | path = [] 83 | if options[:add_client_prefix] 84 | path << settings[:client][:name] 85 | end 86 | path << options[:path_prefix] 87 | path = (path + args).join('.') 88 | @metrics << [path, value, Time.now.to_i].join(' ') 89 | end 90 | 91 | def read_file(file_path, chunk_size = nil) 92 | content = '' 93 | File.open(file_path, 'r') do |file| 94 | read_chunk = Proc.new do 95 | content << file.read(chunk_size) 96 | unless file.eof? 97 | EM.next_tick(read_chunk) 98 | else 99 | yield content 100 | end 101 | end 102 | read_chunk.call 103 | end 104 | end 105 | 106 | def parse_proc_stat 107 | sample = {} 108 | cpu_metrics = ['user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq', 'steal', 'guest'] 109 | misc_metrics = ['ctxt', 'processes', 'procs_running', 'procs_blocked', 'btime', 'intr'] 110 | read_file('/proc/stat') do |proc_stat| 111 | proc_stat.each_line do |line| 112 | next if line.empty? 113 | data = line.split(/\s+/) 114 | key = data.shift 115 | values = data.map(&:to_i) 116 | if key =~ /cpu([0-9]+|)/ 117 | sample[key] = Hash[cpu_metrics.zip(values)] 118 | sample[key]['total'] = values.inject(:+) 119 | elsif misc_metrics.include?(key) 120 | sample[key] = values.last 121 | end 122 | end 123 | yield sample 124 | end 125 | end 126 | 127 | def proc_stat_metrics 128 | parse_proc_stat do |previous| 129 | EM::Timer.new(1) do 130 | parse_proc_stat do |sample| 131 | sample.each do |key, data| 132 | if key =~ /^cpu/ 133 | cpu_total_diff = (data.delete('total') - previous[key]['total']) + 1 134 | data.each do |metric, value| 135 | next if value.nil? 136 | diff = value - previous[key][metric] 137 | used = sprintf('%.02f', (diff / cpu_total_diff.to_f) * 100) 138 | add_metric(key, metric, used) 139 | end 140 | else 141 | add_metric(key, data) 142 | end 143 | end 144 | yield 145 | end 146 | end 147 | end 148 | end 149 | 150 | def proc_loadavg_metrics 151 | read_file('/proc/loadavg') do |proc_loadavg| 152 | values = proc_loadavg.split(/\s+/).take(3).map(&:to_f) 153 | add_metric('load_avg', '1_min', values[0]) 154 | add_metric('load_avg', '5_min', values[1]) 155 | add_metric('load_avg', '15_min', values[2]) 156 | yield 157 | end 158 | end 159 | 160 | def proc_net_dev_metrics 161 | dev_metrics = [ 162 | 'rxBytes', 'rxPackets', 'rxErrors', 'rxDrops', 163 | 'rxFifo', 'rxFrame', 'rxCompressed', 'rxMulticast', 164 | 'txBytes', 'txPackets', 'txErrors', 'txDrops', 165 | 'txFifo', 'txColls', 'txCarrier', 'txCompressed' 166 | ] 167 | read_file('/proc/net/dev') do |proc_net_dev| 168 | proc_net_dev.each_line do |line| 169 | interface, data = line.scan(/^\s*([^:]+):\s*(.*)$/).first 170 | next unless interface 171 | values = data.split(/\s+/).map(&:to_i) 172 | Hash[dev_metrics.zip(values)].each do |key, value| 173 | add_metric('net', interface, key.downcase, value) 174 | end 175 | end 176 | yield 177 | end 178 | end 179 | 180 | def proc_meminfo_metrics 181 | read_file('/proc/meminfo') do |proc_meminfo| 182 | proc_meminfo.each_line do |line| 183 | next if line.strip.empty? 184 | root, data = line.split(':') 185 | values = data.strip.split(/\s+/).map(&:to_i) 186 | case root 187 | when /Mem\w+/, 'Buffers', 'Cached', 'Active', 'Committed_AS' 188 | key = root.gsub(/^Mem/, '').downcase 189 | add_metric('memory', key, values.first) 190 | when /Swap\w+/ 191 | key = root.gsub(/^Swap/, '').downcase 192 | add_metric('swap', key, values.first) 193 | end 194 | end 195 | yield 196 | end 197 | end 198 | end 199 | end 200 | end 201 | -------------------------------------------------------------------------------- /files/default/plugins/check-procs.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # 3 | # Check Procs 4 | # === 5 | # 6 | # Finds processes matching various filters (name, state, etc). Will not 7 | # match itself by default. The number of processes found will be tested 8 | # against the Warning/critical thresholds. By default, fails with a 9 | # CRITICAL if more than one process matches -- you must specify values 10 | # for -w and -c to override this. 11 | # 12 | # Attempts to work on Cygwin (where ps does not have the features we 13 | # need) by calling Windows' tasklist.exe, but this is not well tested. 14 | # 15 | # Examples: 16 | # 17 | # # chef-client is running 18 | # check-procs -p chef-client -W 1 19 | # 20 | # # there are not too many zombies 21 | # check-procs -s Z -w 5 -c 10 22 | # 23 | # Copyright 2011 Sonian, Inc 24 | # 25 | # Released under the same terms as Sensu (the MIT license); see LICENSE 26 | # for details. 27 | 28 | require 'rubygems' if RUBY_VERSION < '1.9.0' 29 | require 'sensu-plugin/check/cli' 30 | 31 | class CheckProcs < Sensu::Plugin::Check::CLI 32 | 33 | def self.read_pid(path) 34 | begin 35 | File.read(path).chomp.to_i 36 | rescue 37 | self.new.unknown "Could not read pid file #{path}" 38 | end 39 | end 40 | 41 | option :warn_over, 42 | :short => '-w N', 43 | :long => '--warn-over N', 44 | :description => 'Trigger a warning if over a number', 45 | :proc => proc {|a| a.to_i } 46 | 47 | option :crit_over, 48 | :short => '-c N', 49 | :long => '--critical-over N', 50 | :description => 'Trigger a critical if over a number', 51 | :proc => proc {|a| a.to_i } 52 | 53 | option :warn_under, 54 | :short => '-W N', 55 | :long => '--warn-under N', 56 | :description => 'Trigger a warning if under a number', 57 | :proc => proc {|a| a.to_i }, 58 | :default => 1 59 | 60 | option :crit_under, 61 | :short => '-C N', 62 | :long => '--critical-under N', 63 | :description => 'Trigger a critial if under a number', 64 | :proc => proc {|a| a.to_i }, 65 | :default => 1 66 | 67 | option :metric, 68 | :short => '-t METRIC', 69 | :long => '--metric METRIC', 70 | :description => 'Trigger a critical if there are METRIC procs', 71 | :proc => proc {|a| a.to_sym } 72 | 73 | option :match_self, 74 | :short => '-m', 75 | :long => '--match-self', 76 | :description => 'Match itself', 77 | :boolean => true, 78 | :default => false 79 | 80 | option :match_parent, 81 | :short => '-M', 82 | :long => '--match-parent', 83 | :description => 'Match parent process it uses ruby {process.ppid}', 84 | :boolean => true, 85 | :default => false 86 | 87 | option :cmd_pat, 88 | :short => '-p PATTERN', 89 | :long => '--pattern PATTERN', 90 | :description => 'Match a command against this pattern' 91 | 92 | option :file_pid, 93 | :short => '-f PID', 94 | :long => '--file-pid PID', 95 | :description => 'Check against a specific PID', 96 | :proc => proc {|a| read_pid(a) } 97 | 98 | option :vsz, 99 | :short => '-z VSZ', 100 | :long => '--virtual-memory-size VSZ', 101 | :description => 'Trigger on a Virtual Memory size is bigger than this', 102 | :proc => proc {|a| a.to_i } 103 | 104 | option :rss, 105 | :short => '-r RSS', 106 | :long => '--resident-set-size RSS', 107 | :description => 'Trigger on a Resident Set size is bigger than this', 108 | :proc => proc {|a| a.to_i } 109 | 110 | option :pcpu, 111 | :short => '-P PCPU', 112 | :long => '--proportional-set-size PCPU', 113 | :description => 'Trigger on a Proportional Set Size is bigger than this', 114 | :proc => proc {|a| a.to_f } 115 | 116 | option :state, 117 | :short => '-s STATE', 118 | :long => '--state STATE', 119 | :description => 'Trigger on a specific state, example: Z for zombie', 120 | :proc => proc {|a| a.split(',') } 121 | 122 | option :user, 123 | :short => '-u USER', 124 | :long => '--user USER', 125 | :description => 'Trigger on a specific user', 126 | :proc => proc {|a| a.split(',') } 127 | 128 | option :esec_over, 129 | :short => '-e SECONDS', 130 | :long => '--esec-over SECONDS', 131 | :proc => proc {|a| a.to_i }, 132 | :description => 'Match processes that older that this, in SECONDS' 133 | 134 | option :esec_under, 135 | :short => '-E SECONDS', 136 | :long => '--esec-under SECONDS', 137 | :proc => proc {|a| a.to_i }, 138 | :description => 'Match process that are younger than this, in SECONDS' 139 | 140 | def read_lines(cmd) 141 | IO.popen(cmd + ' 2>&1') do |child| 142 | child.read.split("\n") 143 | end 144 | end 145 | 146 | def line_to_hash(line, *cols) 147 | Hash[cols.zip(line.strip.split(/\s+/, cols.size))] 148 | end 149 | 150 | def on_cygwin? 151 | `ps -W 2>&1`; $?.exitstatus == 0 152 | end 153 | 154 | def get_procs 155 | if on_cygwin? 156 | read_lines('ps -aWl').drop(1).map do |line| 157 | # Horrible hack because cygwin's ps has no o option, every 158 | # format includes the STIME column (which may contain spaces), 159 | # and the process state (which isn't actually a column) can be 160 | # blank. As of revision 1.35, the format is: 161 | # const char *lfmt = "%c %7d %7d %7d %10u %4s %4u %8s %s\n"; 162 | state = line.slice!(0..0) 163 | _stime = line.slice!(45..53) 164 | line_to_hash(line, :pid, :ppid, :pgid, :winpid, :tty, :uid, :etime, :command).merge(:state => state) 165 | end 166 | else 167 | read_lines('ps axwwo user,pid,vsz,rss,pcpu,state,etime,command').drop(1).map do |line| 168 | line_to_hash(line, :user, :pid, :vsz, :rss, :pcpu, :state, :etime, :command) 169 | end 170 | end 171 | end 172 | 173 | def etime_to_esec(etime) 174 | m = /(\d+-)?(\d\d:)?(\d\d):(\d\d)/.match(etime) 175 | (m[1]||0).to_i*86400 + (m[2]||0).to_i*3600 + (m[3]||0).to_i*60 + (m[4]||0).to_i 176 | end 177 | 178 | def run 179 | procs = get_procs 180 | 181 | procs.reject! {|p| p[:pid].to_i != config[:file_pid] } if config[:file_pid] 182 | procs.reject! {|p| p[:pid].to_i == $$ } unless config[:match_self] 183 | procs.reject! {|p| p[:pid].to_i == Process.ppid } unless config[:match_parent] 184 | procs.reject! {|p| p[:command] !~ /#{config[:cmd_pat]}/ } if config[:cmd_pat] 185 | procs.reject! {|p| p[:vsz].to_f < config[:vsz] } if config[:vsz] 186 | procs.reject! {|p| p[:rss].to_f < config[:rss] } if config[:rss] 187 | procs.reject! {|p| p[:pcpu].to_f < config[:pcpu] } if config[:pcpu] 188 | procs.reject! {|p| etime_to_esec(p[:etime]) >= config[:esec_under] } if config[:esec_under] 189 | procs.reject! {|p| etime_to_esec(p[:etime]) <= config[:esec_over] } if config[:esec_over] 190 | procs.reject! {|p| !config[:state].include?(p[:state]) } if config[:state] 191 | procs.reject! {|p| !config[:user].include?(p[:user]) } if config[:user] 192 | 193 | msg = "Found #{procs.size} matching processes" 194 | msg += "; cmd /#{config[:cmd_pat]}/" if config[:cmd_pat] 195 | msg += "; state #{config[:state].join(',')}" if config[:state] 196 | msg += "; user #{config[:user].join(',')}" if config[:user] 197 | msg += "; vsz > #{config[:vsz]}" if config[:vsz] 198 | msg += "; rss > #{config[:rss]}" if config[:rss] 199 | msg += "; pcpu > #{config[:pcpu]}" if config[:pcpu] 200 | msg += "; esec < #{config[:esec_under]}" if config[:esec_under] 201 | msg += "; esec > #{config[:esec_over]}" if config[:esec_over] 202 | msg += "; pid #{config[:file_pid]}" if config[:file_pid] 203 | 204 | if config[:metric] 205 | count = procs.map {|p| p[config[:metric]].to_i }.reduce {|a, b| a + b } 206 | msg += "; #{config[:metric]} == #{count}" 207 | else 208 | count = procs.size 209 | end 210 | 211 | if !!config[:crit_under] && count < config[:crit_under] 212 | critical msg 213 | elsif !!config[:crit_over] && count > config[:crit_over] 214 | critical msg 215 | elsif !!config[:warn_under] && count < config[:warn_under] 216 | warning msg 217 | elsif !!config[:warn_over] && count > config[:warn_over] 218 | warning msg 219 | else 220 | ok msg 221 | end 222 | end 223 | 224 | end 225 | --------------------------------------------------------------------------------