├── README
├── json-to-graphite.js
├── collectd-graphite-proxy.js
└── types.db


/README:
--------------------------------------------------------------------------------
 1 | 
 2 | Graphite is great.
 3 | 
 4 | Collectd has piles of plugins.
 5 | 
 6 | Let's glue them together.
 7 | 
 8 | In collectd.conf, put this:
 9 | 
10 | <Plugin write_http>
11 |   <URL "http://monitor:3012/post-collectd">
12 |   </URL>
13 | </Plugin>
14 | 
15 | Then run collectd-graphite-proxy.js in node, with the hostname of the graphite
16 | system to target as a command-line parameter. It will take data from collectd
17 | and ship it to graphite.
18 | 
19 | Versions of nodejs reported to work:
20 | - 0.2.0
21 | - 0.4.0
22 | 
23 | This is a work in progress. It also may not stay in node/javascript since I
24 | am not really sold on the platform. I have only run this on node 0.2.0 and
25 | haven't bothered upgrading due to the ever volatile, moving target that nodejs
26 | is. If you find it does not work in newer versions, I'm happy to take patches
27 | to resolve such a problem.
28 | 
29 | Future?
30 | - Support the binary protocol collectd uses for the network plugin.
31 | - Use node-ffi to call collectd plugins (as collectd would) and ship out
32 |   graphite (or any other protocol!)
33 | - Implement as collectd plugin? The collectd plugin docs basically "do not use
34 |   this api" and also requires GPL2, so maybe this route isn't the right one.
35 | 


--------------------------------------------------------------------------------
/json-to-graphite.js:
--------------------------------------------------------------------------------
 1 | var net = require("net");
 2 | 
 3 | function to_dotted_notation(obj, parent_key, result) {
 4 |   if (typeof(parent_key) == 'undefined') parent_key = "";
 5 |   if (typeof(result) == 'undefined') result = {};
 6 | 
 7 |   if (typeof(obj) == 'object') {
 8 |     for (var i in obj) {
 9 |       var key = parent_key ? (parent_key + "." + i) : i;
10 |       to_dotted_notation(obj[i], key, result);
11 |     }
12 |   } else if (typeof(obj) == 'number') {
13 |     result[parent_key] = obj;
14 |   }
15 |   return result;
16 | }
17 | 
18 | var stdin = process.openStdin();
19 | var input = "";
20 | stdin.on("data", function(chunk) {
21 |   input += chunk;
22 | });
23 | stdin.on("end", function() {
24 |   data = JSON.parse(input);
25 |   results = to_dotted_notation(data);
26 | 
27 |   /* TODO(sissel): validate args */
28 |   var address = process.argv[2].split(":");
29 |   address.push(2003); /* default port */
30 |   var host = address[0];
31 |   var port = address[1];
32 |   /* Only fetch matching keys */
33 |   var args = process.argv.slice(3); /* argv[0] == 'node', argv[1] is script name */
34 | 
35 |   /* Create a regexp of (arg)|(arg)|(arg)... */
36 |   var pattern = args.map(function(arg) { return "(" + arg + ")" }).join("|");
37 |   var re = new RegExp(pattern);
38 | 
39 |   var now = Math.floor((new Date()).getTime() / 1000);
40 |   var messages = []
41 |   for (var key in results) { 
42 |     if (re.test(key)) {
43 |       messages.push([key, results[key], now].join(" "));
44 |     }
45 |   }
46 | 
47 |   var graphite = net.createConnection(port, host);
48 |   graphite.on('connect', function() {
49 |     for (var i in messages) {
50 |       graphite.write(messages[i].toLowerCase() + "\n");
51 |     }
52 |     graphite.end();
53 |   });
54 | });
55 | 


--------------------------------------------------------------------------------
/collectd-graphite-proxy.js:
--------------------------------------------------------------------------------
  1 | /* TODO(sissel): make connections retry/etc 
  2 |  * TODO(sissel): make graphite target configurable via command line
  3 |  *
  4 |  * This code is a work in progress.
  5 |  *
  6 |  * To use this, put the following in your collectd config:
  7 |  *
  8 |  * LoadPlugin write_http
  9 |  * <Plugin write_http>
 10 |  *   <URL "http://monitor:3012/post-collectd">
 11 |  *   </URL>
 12 |  * </Plugin>
 13 |  *
 14 |  * This will make collectd write 'PUTVAL' statements over HTTP to the above URL.
 15 |  * This code below will then convert the PUTVAL statements to graphite metrics
 16 |  * and ship them to 'monitor:2003'
 17 |  */
 18 | var http = require("http");
 19 | var net = require("net");
 20 | var assert = require("assert");
 21 | var fs = require('fs');
 22 | 
 23 | var types = fs.readFileSync('/usr/share/collectd/types.db', encoding='utf8').split("\n");
 24 | 
 25 | var typesObj = new Object;
 26 | 
 27 | var type_comments_re = /^#/;
 28 | var type_cut_re = /^([^\s]+)\s+(.*)/;
 29 | 
 30 | for (var i in types) {
 31 |   if (!type_comments_re.exec(types[i])) {
 32 |     typeSet = type_cut_re.exec(types[i])
 33 |     if (!typeSet) { continue; }
 34 |     for (var t=0;t < typeSet.length;t++) {
 35 |       var name = typeSet[1];
 36 |       typesObj[name] = new Array();
 37 |       var eachType = typeSet[2].split(", ")
 38 |       for (var u=0; u < eachType.length; u++){
 39 |         var theName = eachType[u].split(":")[0];
 40 |         typesObj[name].push(theName);
 41 |       }
 42 |     }
 43 |   }
 44 | }
 45 | 
 46 | 
 47 | try {
 48 |   var graphite_connection = net.createConnection(2003, host=process.argv[2]);
 49 | } catch (error) {
 50 |   throw error;
 51 | }
 52 | graphite_connection.on("close", function() {
 53 |   throw new Error("Connection closed");
 54 | });
 55 | graphite_connection.on("error", function() {
 56 |   throw new Error("Connection error");
 57 | });
 58 | 
 59 | var request_handler = function(request, response) {
 60 |   var putval_re = /^PUTVAL ([^ ]+)(?: ([^ ]+=[^ ]+)?) ([0-9.]+)(:.*)/;
 61 |   request.addListener("data", function(chunk) {
 62 |     metrics = chunk.toString().split("\n");
 63 |     for (var i in metrics) {
 64 |       var m = putval_re.exec(metrics[i]);
 65 |       if (!m) {
 66 |         continue;
 67 |       }
 68 |       var values = m[4].split(":");
 69 | 
 70 |       for (var v in values) {
 71 |         
 72 |         var name = m[1];
 73 |         var options = m[2];
 74 |         var time = m[3];
 75 | 
 76 |         if ( v == 0 ) {
 77 |           continue;
 78 |         }
 79 | 
 80 |         // Replace some chars for graphite, split into parts
 81 |         var name_parts = name.replace(/\./g, "_").replace(/\//g, ".").split(".");
 82 | 
 83 |         // Start to construct the new name
 84 |         var rebuild = ["agents"]
 85 | 
 86 |         var host = name_parts[0].split(/_/)[0]
 87 |         rebuild = rebuild.concat(host)
 88 | 
 89 |         // Pluigin names can contain an "instance" which is set apart by a dash
 90 |         var plugin = name_parts[1].split("-")
 91 |         rebuild = rebuild.concat(plugin[0])
 92 |         if (plugin.length > 1) {
 93 |           var plugin_instance = plugin.slice(1).join("-")
 94 |           rebuild = rebuild.concat(plugin_instance)
 95 |         }
 96 |         plugin = plugin[0]
 97 | 
 98 |         // Type names can also contain an "instance"
 99 |         var type = name_parts[2].split("-")
100 |         if (type[0] != plugin) {
101 |           // If type and plugin are equal, delete one to clean up a bit
102 |           rebuild = rebuild.concat(type[0])
103 |         }
104 |         if (type.length > 1) {
105 |           var type_instance = type.slice(1).join("-")
106 |           rebuild = rebuild.concat(type_instance)
107 |         }
108 |         type = type[0]
109 | 
110 |         // Put the name back together
111 |         name = rebuild.join(".")
112 |         
113 |         if ( values.length > 2 ) {
114 |           var metric = name_parts[2];
115 | 
116 |           //  If the metric contains a '-' (after removing the instance name)
117 |           //  then we want to remove it before looking up in the types.db
118 |           index = metric.search(/-/)
119 |           if (index > -1) {
120 |             metric = /^([\w]+)-(.*)$/.exec(metric);
121 |           } else {
122 |             // Kinda a hack
123 |             metric = [ "", metric]
124 |           }
125 |           name = name + "." + typesObj[metric[1]][v - 1];
126 |         }
127 |         message = [name, values[v], time].join(" ");
128 |         graphite_connection.write(message + "\n");
129 | 
130 |       }
131 | 
132 |     }
133 |   });
134 | 
135 |   request.addListener("end", function() {
136 |     response.writeHead(200, {"Content-Type": "text/plain"});
137 |     response.write("OK");
138 |     response.end();
139 |   });
140 | }
141 | 
142 | var server = http.createServer()
143 | server.addListener("request", request_handler)
144 | server.listen(3012);
145 | 


--------------------------------------------------------------------------------
/types.db:
--------------------------------------------------------------------------------
  1 | absolute		value:ABSOLUTE:0:U
  2 | apache_bytes		value:DERIVE:0:U
  3 | apache_connections	value:GAUGE:0:65535
  4 | apache_idle_workers	value:GAUGE:0:65535
  5 | apache_requests		value:DERIVE:0:U
  6 | apache_scoreboard	value:GAUGE:0:65535
  7 | ath_nodes		value:GAUGE:0:65535
  8 | ath_stat		value:DERIVE:0:U
  9 | bitrate			value:GAUGE:0:4294967295
 10 | bytes			value:GAUGE:0:U
 11 | cache_operation		value:DERIVE:0:U
 12 | cache_ratio		value:GAUGE:0:100
 13 | cache_result		value:DERIVE:0:U
 14 | cache_size		value:GAUGE:0:4294967295
 15 | charge			value:GAUGE:0:U
 16 | compression_ratio	value:GAUGE:0:2
 17 | compression		uncompressed:DERIVE:0:U, compressed:DERIVE:0:U
 18 | connections		value:DERIVE:0:U
 19 | conntrack		value:GAUGE:0:4294967295
 20 | contextswitch		value:DERIVE:0:U
 21 | counter			value:COUNTER:U:U
 22 | cpufreq			value:GAUGE:0:U
 23 | cpu			value:DERIVE:0:U
 24 | current_connections	value:GAUGE:0:U
 25 | current_sessions	value:GAUGE:0:U
 26 | current			value:GAUGE:U:U
 27 | delay			value:GAUGE:-1000000:1000000
 28 | derive			value:DERIVE:0:U
 29 | df_complex		value:GAUGE:0:U
 30 | df_inodes		value:GAUGE:0:U
 31 | df			used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623
 32 | disk_latency		read:GAUGE:0:U, write:GAUGE:0:U
 33 | disk_merged		read:DERIVE:0:U, write:DERIVE:0:U
 34 | disk_octets		read:DERIVE:0:U, write:DERIVE:0:U
 35 | disk_ops_complex	value:DERIVE:0:U
 36 | disk_ops		read:DERIVE:0:U, write:DERIVE:0:U
 37 | disk_time		read:DERIVE:0:U, write:DERIVE:0:U
 38 | dns_answer		value:DERIVE:0:U
 39 | dns_notify		value:DERIVE:0:U
 40 | dns_octets		queries:DERIVE:0:U, responses:DERIVE:0:U
 41 | dns_opcode		value:DERIVE:0:U
 42 | dns_qtype_cached	value:GAUGE:0:4294967295
 43 | dns_qtype		value:DERIVE:0:U
 44 | dns_query		value:DERIVE:0:U
 45 | dns_question		value:DERIVE:0:U
 46 | dns_rcode		value:DERIVE:0:U
 47 | dns_reject		value:DERIVE:0:U
 48 | dns_request		value:DERIVE:0:U
 49 | dns_resolver		value:DERIVE:0:U
 50 | dns_response		value:DERIVE:0:U
 51 | dns_transfer		value:DERIVE:0:U
 52 | dns_update		value:DERIVE:0:U
 53 | dns_zops		value:DERIVE:0:U
 54 | email_check		value:GAUGE:0:U
 55 | email_count		value:GAUGE:0:U
 56 | email_size		value:GAUGE:0:U
 57 | entropy			value:GAUGE:0:4294967295
 58 | fanspeed		value:GAUGE:0:U
 59 | file_size		value:GAUGE:0:U
 60 | files			value:GAUGE:0:U
 61 | fork_rate		value:DERIVE:0:U
 62 | frequency		value:GAUGE:0:U
 63 | frequency_offset	value:GAUGE:-1000000:1000000
 64 | fscache_stat		value:DERIVE:0:U
 65 | gauge			value:GAUGE:U:U
 66 | http_request_methods	value:DERIVE:0:U
 67 | http_requests		value:DERIVE:0:U
 68 | http_response_codes	value:DERIVE:0:U
 69 | humidity		value:GAUGE:0:100
 70 | if_collisions		value:DERIVE:0:U
 71 | if_dropped		rx:DERIVE:0:U, tx:DERIVE:0:U
 72 | if_errors		rx:DERIVE:0:U, tx:DERIVE:0:U
 73 | if_multicast		value:DERIVE:0:U
 74 | if_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
 75 | if_packets		rx:DERIVE:0:U, tx:DERIVE:0:U
 76 | if_rx_errors		value:DERIVE:0:U
 77 | if_tx_errors		value:DERIVE:0:U
 78 | invocations		value:DERIVE:0:U
 79 | io_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
 80 | io_packets		rx:DERIVE:0:U, tx:DERIVE:0:U
 81 | ipt_bytes		value:DERIVE:0:U
 82 | ipt_packets		value:DERIVE:0:U
 83 | irq			value:DERIVE:0:U
 84 | latency			value:GAUGE:0:65535
 85 | links			value:GAUGE:0:U
 86 | load			shortterm:GAUGE:0:100, midterm:GAUGE:0:100, longterm:GAUGE:0:100
 87 | memcached_command	value:DERIVE:0:U
 88 | memcached_connections	value:GAUGE:0:U
 89 | memcached_items		value:GAUGE:0:U
 90 | memcached_octets	rx:DERIVE:0:U, tx:DERIVE:0:U
 91 | memcached_ops		value:DERIVE:0:U
 92 | memory			value:GAUGE:0:281474976710656
 93 | multimeter		value:GAUGE:U:U
 94 | mysql_commands		value:DERIVE:0:U
 95 | mysql_handler		value:DERIVE:0:U
 96 | mysql_locks		value:DERIVE:0:U
 97 | mysql_log_position	value:DERIVE:0:U
 98 | mysql_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
 99 | nfs_procedure		value:DERIVE:0:U
100 | nginx_connections	value:GAUGE:0:U
101 | nginx_requests		value:DERIVE:0:U
102 | node_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
103 | node_rssi		value:GAUGE:0:255
104 | node_stat		value:DERIVE:0:U
105 | node_tx_rate		value:GAUGE:0:127
106 | operations		value:DERIVE:0:U
107 | percent			value:GAUGE:0:100.1
108 | pg_blks			value:DERIVE:0:U
109 | pg_db_size		value:GAUGE:0:U
110 | pg_n_tup_c		value:DERIVE:0:U
111 | pg_n_tup_g		value:GAUGE:0:U
112 | pg_numbackends		value:GAUGE:0:U
113 | pg_scan			value:DERIVE:0:U
114 | pg_xact			value:DERIVE:0:U
115 | ping_droprate		value:GAUGE:0:100
116 | ping			value:GAUGE:0:65535
117 | ping_stddev		value:GAUGE:0:65535
118 | players			value:GAUGE:0:1000000
119 | power			value:GAUGE:0:U
120 | protocol_counter	value:DERIVE:0:U
121 | ps_code			value:GAUGE:0:9223372036854775807
122 | ps_count		processes:GAUGE:0:1000000, threads:GAUGE:0:1000000
123 | ps_cputime		user:DERIVE:0:U, syst:DERIVE:0:U
124 | ps_data			value:GAUGE:0:9223372036854775807
125 | ps_disk_octets		read:DERIVE:0:U, write:DERIVE:0:U
126 | ps_disk_ops		read:DERIVE:0:U, write:DERIVE:0:U
127 | ps_pagefaults		minflt:DERIVE:0:U, majflt:DERIVE:0:U
128 | ps_rss			value:GAUGE:0:9223372036854775807
129 | ps_stacksize		value:GAUGE:0:9223372036854775807
130 | ps_state		value:GAUGE:0:65535
131 | ps_vm			value:GAUGE:0:9223372036854775807
132 | queue_length		value:GAUGE:0:U
133 | records			value:GAUGE:0:U
134 | requests		value:GAUGE:0:U
135 | response_time		value:GAUGE:0:U
136 | route_etx		value:GAUGE:0:U
137 | route_metric		value:GAUGE:0:U
138 | routes			value:GAUGE:0:U
139 | serial_octets		rx:DERIVE:0:U, tx:DERIVE:0:U
140 | signal_noise		value:GAUGE:U:0
141 | signal_power		value:GAUGE:U:0
142 | signal_quality		value:GAUGE:0:U
143 | snr			value:GAUGE:0:U
144 | spam_check		value:GAUGE:0:U
145 | spam_score		value:GAUGE:U:U
146 | swap_io			value:DERIVE:0:U
147 | swap			value:GAUGE:0:1099511627776
148 | tcp_connections		value:GAUGE:0:4294967295
149 | temperature		value:GAUGE:-273.15:U
150 | threads			value:GAUGE:0:U
151 | time_dispersion		value:GAUGE:-1000000:1000000
152 | timeleft		value:GAUGE:0:3600
153 | time_offset		value:GAUGE:-1000000:1000000
154 | total_bytes		value:DERIVE:0:U
155 | total_connections	value:DERIVE:0:U
156 | total_operations	value:DERIVE:0:U
157 | total_requests		value:DERIVE:0:U
158 | total_sessions		value:DERIVE:0:U
159 | total_threads		value:DERIVE:0:U
160 | total_time_in_ms	value:DERIVE:0:U
161 | total_values		value:DERIVE:0:U
162 | uptime			value:GAUGE:0:4294967295
163 | users			value:GAUGE:0:65535
164 | vcpu			value:GAUGE:0:U
165 | virt_cpu_total		value:DERIVE:0:U
166 | virt_vcpu		value:DERIVE:0:U
167 | vmpage_action		value:DERIVE:0:U
168 | vmpage_faults		minflt:DERIVE:0:U, majflt:DERIVE:0:U
169 | vmpage_io		in:DERIVE:0:U, out:DERIVE:0:U
170 | vmpage_number		value:GAUGE:0:4294967295
171 | volatile_changes	value:GAUGE:0:U
172 | voltage_threshold	value:GAUGE:U:U, threshold:GAUGE:U:U
173 | voltage			value:GAUGE:U:U
174 | vs_memory		value:GAUGE:0:9223372036854775807
175 | vs_processes		value:GAUGE:0:65535
176 | vs_threads		value:GAUGE:0:65535
177 | #
178 | # Legacy types
179 | # (required for the v5 upgrade target)
180 | #
181 | arc_counts		demand_data:COUNTER:0:U, demand_metadata:COUNTER:0:U, prefetch_data:COUNTER:0:U, prefetch_metadata:COUNTER:0:U
182 | arc_l2_bytes		read:COUNTER:0:U, write:COUNTER:0:U
183 | arc_l2_size		value:GAUGE:0:U
184 | arc_ratio		value:GAUGE:0:U
185 | arc_size		current:GAUGE:0:U, target:GAUGE:0:U, minlimit:GAUGE:0:U, maxlimit:GAUGE:0:U
186 | mysql_qcache		hits:COUNTER:0:U, inserts:COUNTER:0:U, not_cached:COUNTER:0:U, lowmem_prunes:COUNTER:0:U, queries_in_cache:GAUGE:0:U
187 | mysql_threads		running:GAUGE:0:U, connected:GAUGE:0:U, cached:GAUGE:0:U, created:COUNTER:0:U
188 | 


--------------------------------------------------------------------------------