├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── lib └── influxdb.js ├── package.json └── scripts ├── packet-generator.js └── setup-influxdb.js /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /statsd-config.js 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | StatsD InfluxDB backend - CHANGELOG 2 | ----------------------------------- 3 | 4 | ## v0.6.0 (2015-06-23) 5 | 6 | * Initial InfluxDB 0.9 API support (#14, #17) 7 | * Make username and password config options optional. (#15) 8 | 9 | ## v0.5.0 (2015-04-02) 10 | 11 | * Unbreak conditional. (#13) 12 | * Updated regex for statsd metrics. (#12) 13 | * Expose internal backend metrics. (#9) 14 | 15 | ## v0.4.1 (2015-02-23) 16 | 17 | * Use prefixStats setting instead of hardcoding prefix. (#6) 18 | 19 | ## v0.4.0 (2015-02-22) 20 | 21 | * Add configuration option to enable sending internal statsd metrics. (#6) 22 | * Add SET support. (#7) 23 | * Fix problem with histograms breaking data flushes. (#8) 24 | * Add SSL support. (#3) 25 | * Improve configuration example in README. 26 | 27 | ## v0.3.0 (2014-08-24) 28 | 29 | * Allow gauges with a value of 0 to be sent. (#2) 30 | * Update `time_precision` to use `ms`. (#1) 31 | 32 | ## v0.2.0 (2013-11-19) 33 | 34 | * Add flush strategy and enable it by default. 35 | * Add configuration options for flush and proxy strategy. 36 | 37 | ## v0.1.0 (2013-11-15) 38 | 39 | * Initial release. 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Bernd Ahlers 2 | 3 | Permission to use, copy, modify, and distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | StatsD InfluxDB backend 2 | ----------------------- 3 | 4 | **LOOKING FOR A MAINTAINER:** [I am looking for a maintainer for this project](https://github.com/bernd/statsd-influxdb-backend/issues/26) 5 | 6 | A naive [InfluxDB](http://influxdb.org/) backend for 7 | [StatsD](https://github.com/etsy/statsd). 8 | 9 | It can ship events to InfluxDB using two different strategies which can be 10 | used at the same time. 11 | 12 | ### Regular Flush Strategy 13 | 14 | StatsD will flush aggregated metrics with a configured interval. This is 15 | the regular StatsD mode of operation. 16 | 17 | ### Proxy Strategy 18 | 19 | This will map every incoming StatsD packet to an InfluxDB event. It's useful 20 | if you want to store the raw events in InfluxDB without any rollups. 21 | 22 | ## CAVEATS 23 | 24 | This is pretty young and I do not have much experience with InfluxDB yet. 25 | Especially the event buffering and the event mapping might be problematic 26 | and inefficient. 27 | 28 | InfluxDB is also pretty young and there might be breaking changes until it 29 | reaches 1.0. 30 | 31 | Please be careful! 32 | 33 | ## Installation 34 | 35 | $ cd /path/to/statsd 36 | $ npm install statsd-influxdb-backend 37 | 38 | ## Configuration 39 | 40 | You can configure the following settings in your StatsD config file. 41 | 42 | ```js 43 | { 44 | graphitePort: 2003, 45 | graphiteHost: "graphite.example.com", 46 | port: 8125, 47 | backends: [ "./backends/graphite", "statsd-influxdb-backend" ], 48 | 49 | influxdb: { 50 | host: '127.0.0.1', // InfluxDB host. (default 127.0.0.1) 51 | port: 8086, // InfluxDB port. (default 8086) 52 | version: 0.8, // InfluxDB version. (default 0.8) 53 | ssl: false, // InfluxDB is hosted over SSL. (default false) 54 | database: 'dbname', // InfluxDB database instance. (required) 55 | username: 'user', // InfluxDB database username. 56 | password: 'pass', // InfluxDB database password. 57 | flush: { 58 | enable: true // Enable regular flush strategy. (default true) 59 | }, 60 | proxy: { 61 | enable: false, // Enable the proxy strategy. (default false) 62 | suffix: 'raw', // Metric name suffix. (default 'raw') 63 | flushInterval: 1000 // Flush interval for the internal buffer. 64 | // (default 1000) 65 | }, 66 | includeStatsdMetrics: false, // Send internal statsd metrics to InfluxDB. (default false) 67 | includeInfluxdbMetrics: false // Send internal backend metrics to InfluxDB. (default false) 68 | // Requires includeStatsdMetrics to be enabled. 69 | } 70 | } 71 | ``` 72 | 73 | ## Activation 74 | 75 | Add the `statsd-influxdb-backend` to the list of StatsD backends in the config 76 | file and restart the StatsD process. 77 | 78 | ```js 79 | { 80 | backends: ['./backends/graphite', 'statsd-influxdb-backend'] 81 | } 82 | ``` 83 | 84 | ## Unsupported Metric Types 85 | 86 | #### Proxy Strategy 87 | 88 | * Counter with sampling. 89 | * Signed gauges. (i.e. `bytes:+4|g`) 90 | * Sets 91 | 92 | ## InfluxDB Event Mapping 93 | 94 | StatsD packets are currently mapped to the following InfluxDB events. This is 95 | a first try and I'm open to suggestions to improve this. 96 | 97 | ### Set 98 | 99 | StatsD package `client_version:1.1|c`, `client_version:1.2|c` as Influx event: 100 | 101 | ```js 102 | [ 103 | { 104 | name: 'visior', 105 | columns: ['value', 'time'], 106 | points: [['1.1', 1384798553000], ['1.2', 1384798553001]] 107 | } 108 | ] 109 | ``` 110 | 111 | If you are using Grafana to visualize a Set, then using this query or 112 | something similar 113 | 114 | ``` 115 | SELECT version, count(version) FROM client_version GROUP BY version, time(1m) 116 | ``` 117 | 118 | Also, to count for the size of unique value, another InfluxDB event is 119 | also pushed 120 | 121 | ```js 122 | [ 123 | { 124 | name: 'visitor_count', 125 | columns: ['value', 'time'], 126 | points: [set.length, 1384798553001] 127 | } 128 | ] 129 | ``` 130 | 131 | ### Counter 132 | 133 | StatsD packet `requests:1|c` as InfluxDB event: 134 | 135 | #### Flush Strategy 136 | 137 | ```js 138 | [ 139 | { 140 | name: 'requests.counter', 141 | columns: ['value', 'time'], 142 | points: [[802, 1384798553000]] 143 | } 144 | ] 145 | ``` 146 | 147 | #### Proxy Strategy 148 | 149 | ```js 150 | [ 151 | { 152 | name: 'requests.counter.raw', 153 | columns: ['value', 'time'], 154 | points: [[1, 1384472029572]] 155 | } 156 | ] 157 | ``` 158 | 159 | ### Timing 160 | 161 | StatsD packet `response_time:170|ms` as InfluxDB event: 162 | 163 | #### Flush Strategy 164 | 165 | ```js 166 | [ 167 | { 168 | name: 'response_time.timer.mean_90', 169 | columns: ['value', 'time'], 170 | points: [[445.25761772853184, 1384798553000]] 171 | }, 172 | { 173 | name: 'response_time.timer.upper_90', 174 | columns: ['value', 'time'], 175 | points: [[905, 1384798553000]] 176 | }, 177 | { 178 | name: 'response_time.timer.sum_90', 179 | columns: ['value', 'time'], 180 | points: [[321476, 1384798553000]] 181 | }, 182 | { 183 | name: 'response_time.timer.std', 184 | columns: ['value', 'time'], 185 | points: [[294.4171159604542, 1384798553000]] 186 | }, 187 | { 188 | name: 'response_time.timer.upper', 189 | columns: ['value', 'time'], 190 | points: [[998, 1384798553000]] 191 | }, 192 | { 193 | name: 'response_time.timer.lower', 194 | columns: ['value', 'time'], 195 | points: [[2, 1384798553000]] 196 | }, 197 | { 198 | name: 'response_time.timer.count', 199 | columns: ['value', 'time'], 200 | points: [[802, 1384798553000]] 201 | }, 202 | { 203 | name: 'response_time.timer.count_ps', 204 | columns: ['value', 'time'], 205 | points: [[80.2, 1384798553000]] 206 | }, 207 | { 208 | name: 'response_time.timer.sum', 209 | columns: ['value', 'time'], 210 | points: [[397501, 1384798553000]] 211 | }, 212 | { 213 | name: 'response_time.timer.mean', 214 | columns: ['value', 'time'], 215 | points: [[495.6371571072319, 1384798553000]] 216 | }, 217 | { 218 | name: 'response_time.timer.median', 219 | columns: ['value', 'time'], 220 | points: [[483, 1384798553000]] 221 | } 222 | ] 223 | ``` 224 | 225 | #### Proxy Strategy 226 | 227 | ```js 228 | [ 229 | { 230 | name: 'response_time.timer.raw', 231 | columns: ['value', 'time'], 232 | points: [[170, 1384472029572]] 233 | } 234 | ] 235 | ``` 236 | 237 | ### Gauges 238 | 239 | StatsD packet `bytes:123|g` as InfluxDB event: 240 | 241 | #### Flush Strategy 242 | 243 | ```js 244 | [ 245 | { 246 | name: 'bytes.gauge', 247 | columns: ['value', 'time'], 248 | points: [[123, 1384798553000]] 249 | } 250 | ] 251 | ``` 252 | 253 | #### Proxy Strategy 254 | 255 | ```js 256 | [ 257 | { 258 | name: 'bytes.gauge.raw', 259 | columns: ['value', 'time'], 260 | points: [['gauge', 123, 1384472029572]] 261 | } 262 | ] 263 | ``` 264 | 265 | ## Proxy Strategy Notes 266 | 267 | ### Event Buffering 268 | 269 | To avoid one HTTP request per StatsD packet, the InfluxDB backend buffers the 270 | incoming events and flushes the buffer on a regular basis. The current default 271 | is 1000ms. Use the `influxdb.proxy.flushInterval` to change the interval. 272 | 273 | This might become a problem with lots of incoming events. 274 | 275 | The payload of a HTTP request might look like this: 276 | 277 | ```js 278 | [ 279 | { 280 | name: 'requests.counter.raw', 281 | columns: ['value', 'time'], 282 | points: [ 283 | [1, 1384472029572], 284 | [1, 1384472029573], 285 | [1, 1384472029580] 286 | ] 287 | }, 288 | { 289 | name: 'response_time.timer.raw', 290 | columns: ['value', 'time'], 291 | points: [ 292 | [170, 1384472029570], 293 | [189, 1384472029572], 294 | [234, 1384472029578], 295 | [135, 1384472029585] 296 | ] 297 | }, 298 | { 299 | name: 'bytes.gauge.raw', 300 | columns: ['value', 'time'], 301 | points: [ 302 | [123, 1384472029572], 303 | [123, 1384472029580] 304 | ] 305 | } 306 | ] 307 | ``` 308 | 309 | ## Backend Metrics 310 | 311 | The following internal metrics are calculated for each flush: 312 | 313 | - `statsd.influxdbStats.flush_time` - Time taken to process a complete flush in ms. Excluding the asynchronous HTTP Post. 314 | - `statsd.influxdbStats.http_response_time` - Response time in ms of the InfluxDB HTTP endpoint when POSTing data. 315 | - `statsd.influxdbStats.payload_size` - The size in bytes of the JSON payload. 316 | - `statsd.influxdbStats.num_stats` - The number of metrics sent to InfluxDB in the last flush. 317 | 318 | These are added to the set of internal statsd metrics. If both `influxdb.includeStatsdMetrics` and `influxdb.includeInfluxdbMetrics` are enabled, then these will be sent to InfluxDB when using the flush strategy. 319 | 320 | The internal metrics can also can be viewed using the `stats` command on the [StatsD TCP Admin Interface](https://github.com/etsy/statsd/blob/master/docs/admin_interface.md) 321 | 322 | ## Contributing 323 | 324 | All contributions are welcome: ideas, patches, documentation, bug reports, 325 | complaints, and even something you drew up on a napkin. 326 | -------------------------------------------------------------------------------- /lib/influxdb.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Flush stats to InfluxDB (http://influxdb.org/) 3 | * 4 | * To enable this backend, include 'statsd-influxdb-backend' in the backends 5 | * configuration array: 6 | * 7 | * backends: ['statsd-influxdb-backend'] 8 | * 9 | * The backend will read the configuration options from the following 10 | * 'influxdb' hash defined in the main statsd config file: 11 | * 12 | * influxdb: { 13 | * host: '127.0.0.1', // InfluxDB host. (default 127.0.0.1) 14 | * port: 8086, // InfluxDB port. (default 8086) 15 | * ssl: false, // InfluxDB is hosted over SSL. (default false) 16 | * database: 'dbname', // InfluxDB database instance. (required) 17 | * username: 'user', // InfluxDB database username. 18 | * password: 'pass', // InfluxDB database password. 19 | * flush: { 20 | * enable: true // Enable regular flush strategy. (default true) 21 | * }, 22 | * proxy: { 23 | * enable: false, // Enable the proxy strategy. (default false) 24 | * suffix: 'raw', // Metric name suffix. (default 'raw') 25 | * flushInterval: 1000 // Flush interval for the internal buffer. 26 | * // (default 1000) 27 | * }, 28 | * includeStatsdMetrics: false, // Send internal statsd metrics to InfluxDB. (default false) 29 | * includeInfluxdbMetrics: false // Send internal backend metrics to InfluxDB. (default false) 30 | * // Requires includeStatsdMetrics to be enabled. 31 | * } 32 | * 33 | */ 34 | var util = require('util'), 35 | querystring = require('querystring'), 36 | http = require('http'), 37 | https = require('https'); 38 | 39 | function InfluxdbBackend(startupTime, config, events) { 40 | var self = this; 41 | 42 | self.debug = config.debug; 43 | self.registry = {}; 44 | self.influxdbStats = {}; 45 | 46 | self.defaultHost = '127.0.0.1'; 47 | self.defaultPort = 8086; 48 | self.defaultVersion = 0.8; 49 | self.defaultFlushEnable = true; 50 | self.defaultProxyEnable = false; 51 | self.defaultProxySuffix = 'raw'; 52 | self.defaultProxyFlushInterval = 1000; 53 | 54 | self.host = self.defaultHost; 55 | self.port = self.defaultPort; 56 | self.version = self.defaultVersion; 57 | self.protocol = http; 58 | self.flushEnable = self.defaultFlushEnable; 59 | self.proxyEnable = self.defaultProxyEnable; 60 | self.proxySuffix = self.defaultProxySuffix; 61 | self.proxyFlushInterval = self.defaultProxyFlushInterval; 62 | self.includeStatsdMetrics = false; 63 | self.includeInfluxdbMetrics = false; 64 | 65 | /* XXX Hardcoding default prefix here because it is not accessible otherwise. */ 66 | self.prefixStats = config.prefixStats !== undefined ? config.prefixStats : 'statsd'; 67 | 68 | if (config.influxdb) { 69 | self.host = config.influxdb.host || self.defaultHost; 70 | self.port = config.influxdb.port || self.defaultPort; 71 | self.version = config.influxdb.version || self.defaultVersion; 72 | self.user = config.influxdb.username; 73 | self.pass = config.influxdb.password; 74 | self.database = config.influxdb.database; 75 | self.includeStatsdMetrics = config.influxdb.includeStatsdMetrics; 76 | self.includeInfluxdbMetrics = config.influxdb.includeInfluxdbMetrics; 77 | 78 | if (config.influxdb.ssl) { 79 | self.protocol = https; 80 | } 81 | 82 | if (config.influxdb.flush) { 83 | self.flushEnable = config.influxdb.flush.enable; 84 | } 85 | 86 | if (config.influxdb.proxy) { 87 | self.proxyEnable = config.influxdb.proxy.enable || self.defaultProxyEnable; 88 | self.proxySuffix = config.influxdb.proxy.suffix || self.defaultProxySuffix; 89 | self.proxyFlushInterval = config.influxdb.proxy.flushInterval || self.defaultProxyFlushInterval; 90 | } 91 | } 92 | 93 | if (self.version >= 0.9) { 94 | self.assembleEvent = self.assembleEvent_v09; 95 | self.httpPOST = self.httpPOST_v09; 96 | } else { 97 | self.assembleEvent = self.assembleEvent_v08; 98 | self.httpPOST = self.httpPOST_v08; 99 | } 100 | 101 | if (self.proxyEnable) { 102 | self.log('Starting the buffer flush interval. (every ' + self.proxyFlushInterval + 'ms)'); 103 | setInterval(function () { 104 | self.flushQueue(); 105 | }, self.proxyFlushInterval); 106 | 107 | events.on('packet', function (packet, rinfo) { 108 | try { 109 | self.processPacket(packet, rinfo); 110 | } catch (e) { 111 | self.log(e); 112 | } 113 | }); 114 | } 115 | 116 | if (self.flushEnable) { 117 | events.on('flush', function (timestamp, metrics) { 118 | try { 119 | self.processFlush(timestamp, metrics); 120 | } catch (e) { 121 | self.log(e); 122 | } 123 | }); 124 | } 125 | 126 | events.on('status', function (writeCb) { 127 | for (var stat in self.influxdbStats) { 128 | writeCb(null, 'influxdb', stat, self.influxdbStats[stat]); 129 | } 130 | }); 131 | 132 | return true; 133 | } 134 | 135 | function millisecondsSince(start) { 136 | diff = process.hrtime(start); 137 | return diff[0] * 1000 + diff[1] / 1000000; 138 | } 139 | 140 | InfluxdbBackend.prototype.log = function (msg) { 141 | util.log('[influxdb] ' + msg); 142 | } 143 | 144 | InfluxdbBackend.prototype.logDebug = function (msg) { 145 | if (this.debug) { 146 | var string; 147 | 148 | if (msg instanceof Function) { 149 | string = msg(); 150 | } else { 151 | string = msg; 152 | } 153 | 154 | util.log('[influxdb] (DEBUG) ' + string); 155 | } 156 | } 157 | 158 | /** 159 | * Flush strategy handler 160 | * 161 | * @param {Number} timestamp 162 | * @param {Object} stats metric 163 | */ 164 | InfluxdbBackend.prototype.processFlush = function (timestamp, metrics) { 165 | var self = this, 166 | counters = metrics.counters, 167 | gauges = metrics.gauges, 168 | timerData = metrics.timer_data, 169 | statsdMetrics = metrics.statsd_metrics, 170 | points = [], 171 | sets = function (vals) { 172 | var ret = {}; 173 | for (var val in vals) { 174 | ret[val] = vals[val].values(); 175 | } 176 | return ret; 177 | }(metrics.sets), 178 | startTime = process.hrtime(), 179 | key, timerKey, 180 | statsPrefixRegexp = new RegExp('^' + self.prefixStats + '\\.'); 181 | 182 | /* Convert timestamp from seconds to milliseconds. */ 183 | timestamp = (timestamp * 1000); 184 | 185 | for (key in counters) { 186 | /* Do not include statsd counters. */ 187 | if (!self.includeStatsdMetrics && key.match(statsPrefixRegexp)) { continue; } 188 | 189 | var value = counters[key], 190 | k = key + '.counter'; 191 | 192 | if (value) { 193 | points.push(self.assembleEvent(k, [{value: value, time: timestamp}])); 194 | } 195 | } 196 | 197 | for (set in sets) { 198 | sets[set].map(function (v) { 199 | points.push(self.assembleEvent(set, [{value: v, time: timestamp}])); 200 | }) 201 | points.push(self.assembleEvent(set + "_count", [{value: sets[set].length, time: timestamp}])); 202 | } 203 | 204 | for (key in gauges) { 205 | /* Do not include statsd gauges. */ 206 | if (!self.includeStatsdMetrics && key.match(statsPrefixRegexp)) { continue; } 207 | 208 | var value = gauges[key], 209 | k = key + '.gauge'; 210 | 211 | if (!isNaN(parseFloat(value)) && isFinite(value)) { 212 | points.push(self.assembleEvent(k, [{value: value, time: timestamp}])); 213 | } 214 | } 215 | 216 | for (key in timerData) { 217 | var timerMetrics = timerData[key]; 218 | 219 | // Try to add histogram data, if it is there: 220 | if (timerMetrics.histogram) { 221 | var histoMetrics = timerMetrics.histogram 222 | , histoKey; 223 | 224 | for (histoKey in histoMetrics) { 225 | var value = histoMetrics[histoKey], 226 | k = key + '.timer.histogram.' + histoKey; 227 | 228 | points.push(self.assembleEvent(k, [{value: value, time: timestamp}])); 229 | } 230 | 231 | // Delete here so it isn't iterated over later: 232 | delete timerMetrics.histogram; 233 | } 234 | 235 | // Iterate over normal metrics: 236 | for (timerKey in timerMetrics) { 237 | var value = timerMetrics[timerKey], 238 | k = key + '.timer' + '.' + timerKey; 239 | 240 | points.push(self.assembleEvent(k, [{value: value, time: timestamp}])); 241 | } 242 | } 243 | 244 | if (self.includeStatsdMetrics) { 245 | // Include backend metrics for the previous flush 246 | if (self.includeInfluxdbMetrics) { 247 | statsdMetrics['influxdbStats.flush_time'] = self.influxdbStats.flushTime; 248 | statsdMetrics['influxdbStats.http_response_time'] = self.influxdbStats.httpResponseTime; 249 | statsdMetrics['influxdbStats.payload_size'] = self.influxdbStats.payloadSize; 250 | statsdMetrics['influxdbStats.num_stats'] = self.influxdbStats.numStats; 251 | } 252 | 253 | for (key in statsdMetrics) { 254 | var value = statsdMetrics[key], 255 | k = self.prefixStats + '.' + key; 256 | 257 | if (!isNaN(parseFloat(value)) && isFinite(value)) { 258 | points.push(self.assembleEvent(k, [{value: value, time: timestamp}])); 259 | } 260 | } 261 | } 262 | 263 | self.httpPOST(points); 264 | self.influxdbStats.flushTime = millisecondsSince(startTime); 265 | } 266 | 267 | InfluxdbBackend.prototype.processPacket = function (packet, rinfo) { 268 | var self = this, 269 | ts = (new Date()).valueOf(); 270 | 271 | /* Stolen from statsd's stats.js. */ 272 | var packet_data = packet.toString(), 273 | metrics; 274 | 275 | if (packet_data.indexOf("\n") > -1) { 276 | metrics = packet_data.split("\n"); 277 | } else { 278 | metrics = [packet_data]; 279 | } 280 | 281 | for (var midx in metrics) { 282 | if (metrics[midx].length === 0) { 283 | continue; 284 | } 285 | var bits = metrics[midx].toString().split(':'); 286 | var key = bits.shift() 287 | .replace(/\s+/g, '_') 288 | .replace(/\//g, '-') 289 | .replace(/[^a-zA-Z_\-0-9\.]/g, ''); 290 | 291 | if (bits.length === 0) { 292 | bits.push("1"); 293 | } 294 | 295 | for (var i = 0; i < bits.length; i++) { 296 | var fields = bits[i].split("|"); 297 | 298 | if (fields[1] === undefined) { 299 | self.log('Bad line: ' + fields + ' in msg "' + metrics[midx] +'"'); 300 | continue; 301 | } 302 | 303 | var metric_type = fields[1].trim(); 304 | 305 | /* Timer */ 306 | if (metric_type === "ms") { 307 | self.enqueue('timer', ts, key, Number(fields[0] || 0)); 308 | /* Gauge */ 309 | } else if (metric_type === "g") { 310 | if (fields[0].match(/^[-+]/)) { 311 | self.logDebug('Sending gauges with +/- is not supported yet.'); 312 | } else { 313 | self.enqueue('gauge', ts, key, Number(fields[0] || 0)); 314 | } 315 | /* Set */ 316 | } else if (metric_type === "s") { 317 | self.logDebug('Sets not supported yet.'); 318 | /* Counter */ 319 | } else { 320 | /* XXX Handle sampling. */ 321 | self.enqueue('counter', ts, key, Number(fields[0] || 1)); 322 | } 323 | } 324 | } 325 | } 326 | 327 | InfluxdbBackend.prototype.enqueue = function (type, ts, key, value) { 328 | var self = this; 329 | 330 | key = key + '.' + type + '.' + self.proxySuffix; 331 | 332 | if (!self.registry[key]) { 333 | self.registry[key] = []; 334 | } 335 | 336 | self.registry[key].push({value: value, time: ts}); 337 | } 338 | 339 | InfluxdbBackend.prototype.flushQueue = function () { 340 | var self = this, 341 | registry = self.clearRegistry(), 342 | points = []; 343 | 344 | for (var key in registry) { 345 | var payload = self.assembleEvent(key, registry[key]); 346 | 347 | self.logDebug(function () { 348 | return 'Flush ' + registry[key].length + ' values for ' + key; 349 | }); 350 | 351 | points.push(payload); 352 | } 353 | 354 | self.httpPOST(points); 355 | 356 | self.logDebug('Queue flushed'); 357 | } 358 | 359 | 360 | InfluxdbBackend.prototype.clearRegistry = function () { 361 | var self = this, 362 | registry = self.registry; 363 | 364 | self.registry = {}; 365 | 366 | return registry; 367 | } 368 | 369 | InfluxdbBackend.prototype.assembleEvent_v08 = function (name, events) { 370 | var self = this; 371 | 372 | var payload = { 373 | name: name, 374 | columns: Object.keys(events[0]), 375 | points: [] 376 | }; 377 | 378 | for (var idx in events) { 379 | var event = events[idx], 380 | points = []; 381 | 382 | for (var cidx in payload.columns) { 383 | var column = payload.columns[cidx]; 384 | 385 | points.push(event[column]); 386 | } 387 | 388 | payload.points.push(points); 389 | } 390 | 391 | return payload; 392 | } 393 | 394 | InfluxdbBackend.prototype.assembleEvent_v09 = function (name, events) { 395 | var self = this; 396 | 397 | var payload = { 398 | measurement: name, 399 | fields: { value: events[0]['value'] } 400 | } 401 | 402 | return payload; 403 | } 404 | 405 | InfluxdbBackend.prototype.httpPOST_v08 = function (points) { 406 | /* Do not send if there are no points. */ 407 | if (!points.length) { return; } 408 | 409 | var self = this, 410 | query = {u: self.user, p: self.pass, time_precision: 'ms'}, 411 | protocolName = self.protocol == http ? 'HTTP' : 'HTTPS', 412 | startTime; 413 | 414 | self.logDebug(function () { 415 | return 'Sending ' + points.length + ' different points via ' + protocolName; 416 | }); 417 | 418 | self.influxdbStats.numStats = points.length; 419 | 420 | var options = { 421 | hostname: self.host, 422 | port: self.port, 423 | path: '/db/' + self.database + '/series?' + querystring.stringify(query), 424 | method: 'POST', 425 | agent: false // Is it okay to use "undefined" here? (keep-alive) 426 | }; 427 | 428 | var req = self.protocol.request(options); 429 | 430 | req.on('socket', function (res) { 431 | startTime = process.hrtime(); 432 | }); 433 | 434 | req.on('response', function (res) { 435 | var status = res.statusCode; 436 | 437 | self.influxdbStats.httpResponseTime = millisecondsSince(startTime); 438 | 439 | if (status !== 200) { 440 | self.log(protocolName + ' Error: ' + status); 441 | } 442 | }); 443 | 444 | req.on('error', function (e, i) { 445 | self.log(e); 446 | }); 447 | 448 | var payload = JSON.stringify(points) 449 | self.influxdbStats.payloadSize = Buffer.byteLength(payload); 450 | 451 | self.logDebug(function () { 452 | var size = (self.influxdbStats.payloadSize / 1024).toFixed(2); 453 | return 'Payload size ' + size + ' KB'; 454 | }); 455 | 456 | req.write(payload); 457 | req.end(); 458 | } 459 | 460 | InfluxdbBackend.prototype.httpPOST_v09 = function (points) { 461 | /* Do not send if there are no points. */ 462 | if (!points.length) { return; } 463 | 464 | var self = this, 465 | query = {u: self.user, p: self.pass}, 466 | protocolName = self.protocol == http ? 'HTTP' : 'HTTPS', 467 | startTime; 468 | 469 | self.logDebug(function () { 470 | return 'Sending ' + points.length + ' different points via ' + protocolName; 471 | }); 472 | 473 | self.influxdbStats.numStats = points.length; 474 | 475 | var options = { 476 | hostname: self.host, 477 | port: self.port, 478 | path: '/write?' + querystring.stringify(query), 479 | method: 'POST', 480 | agent: false // Is it okay to use "undefined" here? (keep-alive) 481 | }; 482 | 483 | var req = self.protocol.request(options); 484 | 485 | req.on('socket', function (res) { 486 | startTime = process.hrtime(); 487 | }); 488 | 489 | req.on('response', function (res) { 490 | var status = res.statusCode; 491 | 492 | self.influxdbStats.httpResponseTime = millisecondsSince(startTime); 493 | 494 | if (status >= 400) { 495 | self.log(protocolName + ' Error: ' + status); 496 | } 497 | }); 498 | 499 | req.on('error', function (e, i) { 500 | self.log(e); 501 | }); 502 | 503 | var payload = JSON.stringify({ 504 | database: self.database, 505 | points: points 506 | }); 507 | 508 | self.influxdbStats.payloadSize = Buffer.byteLength(payload); 509 | 510 | self.logDebug(function () { 511 | var size = (self.influxdbStats.payloadSize / 1024).toFixed(2); 512 | return 'Payload size ' + size + ' KB'; 513 | }); 514 | 515 | req.write(payload); 516 | req.end(); 517 | } 518 | 519 | InfluxdbBackend.prototype.configCheck = function () { 520 | var self = this, 521 | success = true; 522 | 523 | /* Make sure the database name is configured. */ 524 | if (!self.database) { 525 | self.log('Missing config option: database'); 526 | success = false; 527 | } 528 | 529 | return success; 530 | } 531 | 532 | exports.init = function (startupTime, config, events) { 533 | var influxdb = new InfluxdbBackend(startupTime, config, events); 534 | 535 | return influxdb.configCheck(); 536 | } 537 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "statsd-influxdb-backend", 3 | "version": "0.6.0", 4 | "description": "InfluxDB backend for StatsD", 5 | "main": "lib/influxdb.js", 6 | "dependencies": { 7 | }, 8 | "scripts": { 9 | "test": "echo \"Error: no test specified\" && exit 1" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git://github.com/bernd/statsd-influxdb-backend.git" 14 | }, 15 | "keywords": [ 16 | "influxdb", 17 | "statsd", 18 | "metrics" 19 | ], 20 | "author": "Bernd Ahlers ", 21 | "license": "BSD", 22 | "bugs": { 23 | "url": "https://github.com/bernd/statsd-influxdb-backend/issues" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /scripts/packet-generator.js: -------------------------------------------------------------------------------- 1 | var dgram = require('dgram'), 2 | socket = dgram.createSocket('udp4'); 3 | 4 | var sendMetric = (function (socket) { 5 | var f = function (msg) { 6 | var message = new Buffer(msg); 7 | 8 | socket.send(message, 0, message.length, 8125, '127.0.0.1', function (e) { 9 | if (e) { console.log(e); } 10 | }); 11 | } 12 | 13 | return f; 14 | })(socket); 15 | 16 | var cnt = 0; 17 | 18 | function sendLoop() { 19 | var random1 = parseInt(Math.random() * 1000), 20 | random2 = parseInt(Math.random() * 1000); 21 | 22 | if ((cnt++ % 100) == 0) { 23 | console.log('count ' + cnt); 24 | } 25 | 26 | sendMetric('api.requests:1|c'); 27 | sendMetric('api.response_times:' + random1 + '|ms'); 28 | sendMetric('api.bytes:' + random2 + '|g'); 29 | } 30 | 31 | setInterval(sendLoop, 1); 32 | -------------------------------------------------------------------------------- /scripts/setup-influxdb.js: -------------------------------------------------------------------------------- 1 | var http = require('http'), 2 | querystring = require('querystring'); 3 | 4 | function httpRequest(method, path, payload) { 5 | var query = {u: 'root', p: 'root'}; 6 | 7 | var req = http.request({ 8 | hostname: 'localhost', 9 | port: 8086, 10 | path: path + '?' + querystring.stringify(query), 11 | method: method.toUpperCase(), 12 | agent: false 13 | }); 14 | 15 | req.on('response', function (res) { 16 | res.on('data', function (chunk) { 17 | console.log('RESPONSE: ' + chunk.toString()); 18 | }); 19 | }); 20 | 21 | req.on('error', function (e) { 22 | console.log(e); 23 | }); 24 | 25 | if (payload) { 26 | req.write(JSON.stringify(payload)); 27 | } 28 | req.end(); 29 | } 30 | 31 | switch (process.argv[2]) { 32 | case 'createdb': { 33 | console.log('Creating database: "statsd"'); 34 | httpRequest('post', '/db', {name: "statsd"}); 35 | break; 36 | } 37 | case 'dropdb': { 38 | console.log('Deleting database: "statsd"'); 39 | httpRequest('delete', '/db/statsd'); 40 | break; 41 | } 42 | case 'createuser': { 43 | console.log('Creating user "user" with password "pass"'); 44 | httpRequest('post', '/db/statsd/users', {username: 'user', password: 'pass'}); 45 | break; 46 | } 47 | case 'createadmin': { 48 | console.log('Creating user "admin" with password "pass"'); 49 | httpRequest('post', '/db/statsd/users', {username: 'admin', password: 'pass', admin: true}); 50 | break; 51 | } 52 | default: { 53 | console.log('Commands: createdb, dropdb, createuser'); 54 | } 55 | } 56 | --------------------------------------------------------------------------------