├── .gitignore ├── LICENSE.md ├── README.md ├── lib └── index.js ├── package.json └── tests ├── exampleFlush.js └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (C) 2012 Dynamic Methods Pty Ltd, David Howell 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StatsD backend for Mongo DB 2 | 3 | ## Overview 4 | This is a pluggable backend for [StatsD](https://github.com/etsy/statsd), which publishes stats to mongodb. 5 | 6 | ## How it works 7 | This backend uses Mongo's capped collections to have near file system performance for logging data points from StatsD. 8 | 9 | ## Installation 10 | 11 | `$ npm install mongo-statsd-backend` 12 | 13 | ## Configuration 14 | 15 | Inside of your StatsD server config file, use the following parameters: 16 | 17 | ```` 18 | { 19 | mongoHost: 'user:pass@localhost', 20 | mongoPort: 27017, 21 | mongoMax: 2160, 22 | mongoPrefix: true, 23 | mongoName: 'databaseName', 24 | backends: ['/path/to/module/lib/index.js'] 25 | } 26 | 27 | ```` 28 | 29 | * `mongoHost`: the ip address or hostname of the mongo server. Default is `localhost`. 30 | * `mongoMax`: the number of data points to cap the collection with. Default is `2160`. With Statsd's default of 10 seconds, this gives 6 hours of 'near real-time' data. 31 | * `mongoPrefix`: Boolean. If true, then the statsd "bucket" names contain a prefix which deterine the database name. For example, if a counter is called 'web-server.page_hits' then the database name will be 'web-server' and the collection name will be 'page_hits'. Otherwise, the database name will be 'statsd' and the collection name will be 'web-server.page_hits'. 32 | 33 | ## Schema 34 | 35 | The schema follows the StatsD namespace. 36 | 37 | `database.collection.tite.flush_rate` 38 | 39 | `bucket.metric.metric_title_flushrate` 40 | 41 | #### `db.counters` 42 | 43 | ```` 44 | { 45 | time: 1234567890, // time_stamp from statsd 46 | count: 124 // Integer from statsd 47 | } 48 | 49 | ```` 50 | 51 | #### `db.timers` 52 | 53 | ```` 54 | { 55 | time: 1234567890, // time_stamp from statsd 56 | durations: [0, 1, 2] // Array from statsd 57 | } 58 | ```` 59 | 60 | #### `db.timer_data` 61 | 62 | ```` 63 | { 64 | time: 1234567890, // time_stamp from statsd 65 | mean_90: 1.1764705882352942, 66 | upper_90: 3, 67 | sum_90: 80, 68 | std: 60.18171889277414, 69 | upper: 526, 70 | lower: 0, 71 | count: 75, 72 | count_ps: 7.5, 73 | sum: 652, 74 | mean: 8.693333333333333, 75 | median: 1 76 | } 77 | ```` 78 | 79 | #### `db.sets` 80 | 81 | ```` 82 | { 83 | time: 1234567890, // time_stamp from statsd 84 | set: ['name1', 'name2'] // Array from statsd 85 | } 86 | ```` 87 | 88 | 89 | 90 | ## Dependencies 91 | - [mongodb](https://github.com/mongodb/node-mongodb-native) 92 | 93 | ## Development 94 | - [Bugs](https://github.com/dynmeth/mongo-statsd-backend/issues) 95 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var mongo = require('mongodb'), 4 | async = require('async'), 5 | util = require('util'), 6 | dbs = {}, 7 | options = { 8 | debug: false, 9 | prefix: true, 10 | size: 100, 11 | max: 2610, 12 | name: 'statsd', 13 | host: '127.0.0.1', 14 | port: 27017 15 | }; 16 | 17 | var connection_queue = async.queue(function(task, callback) { 18 | if(dbs[task.name]) { 19 | callback(null, dbs[task.name]); 20 | } else { 21 | new mongo.Db(task.name, new mongo.Server(options.host, options.port)).open(function(err, db) { 22 | if (err) { 23 | callback(err); 24 | } else { 25 | dbs[task.name] = db; 26 | callback(null, db); 27 | } 28 | }); 29 | } 30 | }, 1); 31 | 32 | var database = function(name, callback) { 33 | if(dbs[name]) { 34 | callback(null, dbs[name]); 35 | } else { 36 | connection_queue.push({name: name}, function(err) { 37 | callback(err, dbs[name]); 38 | }); 39 | } 40 | }; 41 | 42 | /** 43 | * Prefix the db correctly 44 | */ 45 | var dbPrefix = function(metric) { 46 | return options.prefix ? metric.split('.')[0] : options.name; 47 | }; 48 | 49 | /** 50 | * Prefix a collection name 51 | */ 52 | var colPrefix = function(metric_type, metric) { 53 | var ary = metric.split('.'); 54 | if (options.prefix) ary.shift(); 55 | ary.unshift(metric_type); 56 | return ary.join('.')+'_'+options.rate; 57 | }; 58 | 59 | /** 60 | * Aggregate the metrics 61 | */ 62 | var aggregate = { 63 | /** 64 | * Aggregate some metrics bro 65 | * @param {Number} time 66 | * @param {Stirng} key 67 | * @param {String} val 68 | */ 69 | gauges: function(time, key, val) { 70 | return { 71 | db: dbPrefix(key), 72 | col: colPrefix('gauges', key), 73 | data: { 74 | time: time, 75 | gauge: val 76 | }, 77 | }; 78 | }, 79 | /** 80 | * Aggregate some timer_data bro 81 | * @param {Number} time 82 | * @param {Stirng} key 83 | * @param {String} vals 84 | */ 85 | timer_data: function(time, key, val) { 86 | val.time = time; 87 | return { 88 | db: dbPrefix(key), 89 | col: colPrefix('timers', key), 90 | data: val 91 | }; 92 | }, 93 | /** 94 | * Aggregate some timers bro 95 | * @param {Number} time 96 | * @param {Stirng} key 97 | * @param {String} vals 98 | */ 99 | timers: function(time, key, val) { 100 | return { 101 | db: dbPrefix(key), 102 | col: colPrefix('timers', key), 103 | data: { 104 | time: time, 105 | durations: val 106 | }, 107 | }; 108 | }, 109 | /** 110 | * Aggregate some counters bro 111 | * @param {Number} time 112 | * @param {Stirng} key 113 | * @param {String} val 114 | */ 115 | counters: function(time, key, val) { 116 | return { 117 | db: dbPrefix(key), 118 | col: colPrefix('counters', key), 119 | data: { 120 | time: time, 121 | count: val 122 | }, 123 | }; 124 | }, 125 | /** 126 | * Aggregate some sets bro 127 | * @param {Number} time 128 | * @param {Stirng} key 129 | * @param {String} val 130 | */ 131 | sets: function(time, key, val) { 132 | return { 133 | db: dbPrefix(key), 134 | col: colPrefix('sets', key), 135 | data: { 136 | time: time, 137 | set: val 138 | }, 139 | }; 140 | } 141 | }; 142 | 143 | /** 144 | * Insert the data to the database 145 | * @method insert 146 | * @param {String} database 147 | * @param {String} collection 148 | * @param {Object} metric 149 | * @param {Function} callback 150 | */ 151 | var insert = function(dbName, collection, metric, callback) { 152 | var colInfo = {capped:true, size:options.size*options.max, max:options.max}; 153 | 154 | database(dbName, function(err, db) { 155 | if (err) { 156 | db.close(); 157 | return callback(err); 158 | }; 159 | 160 | db.createCollection(collection, colInfo, function(err, collClient) { 161 | collClient.insert(metric, function(err, data){ 162 | if (err) callback(err); 163 | if (!err) callback(false, collection); 164 | }); 165 | }); 166 | }); 167 | }; 168 | 169 | /** 170 | * our `flush` event handler 171 | */ 172 | var onFlush = function(time, metrics) { 173 | var metricTypes = ['gauges', 'timer_data', 'timers', 'counters', 'sets']; 174 | 175 | metricTypes.forEach(function(type, i){ 176 | var obj; 177 | 178 | for (var key in metrics[type]) { 179 | obj = aggregate[type](time, key, metrics[type][key]); 180 | 181 | insert(obj.db, obj.col, obj.data, function(err){ 182 | if (err) console.log(err); 183 | }); 184 | }; 185 | }); 186 | }; 187 | 188 | /** 189 | * Expose our init function to StatsD 190 | * @param {Number} startup_time 191 | * @param {Object} config 192 | * @param {Object} events 193 | */ 194 | exports.init = function(startup_time, config, events) { 195 | if (!startup_time || !config || !events) return false; 196 | 197 | options.debug = config.debug; 198 | 199 | if (typeof config.mongoPrefix == 'boolean' && typeof config.mongoName !== 'string') { 200 | console.log('config.mongoPrefix is false, config.mongoName must be set.'); 201 | return false; 202 | }; 203 | 204 | options.rate = parseInt(config.flushInterval/1000, 10); 205 | options.max = config.mongoMax ? parseInt(config.mongoMax, 10) : 2160; 206 | options.host = config.mongoHost || '127.0.0.1'; 207 | options.prefix = typeof config.mongoPrefix == 'boolean' ? config.mongoPrefix : true; 208 | options.name = config.mongoName; 209 | options.port = config.mongoPort || options.port; 210 | 211 | events.on('flush', onFlush); 212 | 213 | return true; 214 | }; 215 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": "David Howell ", 3 | "description": "A backend for StatsD to emit stats to mongodb.", 4 | "name": "mongo-statsd-backend", 5 | "version": "0.0.6", 6 | "main": "lib/index.js", 7 | "dependencies": { 8 | "mongodb": ">=1.0", 9 | "async": "*" 10 | }, 11 | "devDependencies": { 12 | "mocha": "*", 13 | "chai": "~1.6.0" 14 | }, 15 | "optionalDependencies": {}, 16 | "engines": { 17 | "node": "*" 18 | }, 19 | "scripts":{ 20 | "test":"mocha tests" 21 | }, 22 | "repository": { 23 | "type": "git", 24 | "url": "https://github.com/dynmeth/mongo-statsd-backend.git" 25 | }, 26 | "keywords": [ 27 | "statsd", 28 | "mongodb" 29 | ], 30 | "license": "MIT" 31 | } 32 | -------------------------------------------------------------------------------- /tests/exampleFlush.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | counters: { 3 | 'statsd.bad_lines_seen': 0, 4 | 'statsd.packets_received': 150, 5 | 'central.api_requests': 75 6 | }, 7 | timers: { 8 | 'central.reponse_time': 9 | [ 0, 3, 3, 3, 3, 10, 13, 14, 526 ] 10 | }, 11 | gauges: { 12 | 'central.request_time': 0, 13 | 'central.request_time1': 1, 14 | 'central.request_time2': 2, 15 | 'central.request_time3': 3, 16 | 'central.request_time4': 4, 17 | 'central.request_time5': 5, 18 | 'central.request_time6': 6 19 | }, 20 | timer_data: { 21 | 'central.reponse_time': { 22 | mean_90: 1.1764705882352942, 23 | upper_90: 3, 24 | sum_90: 80, 25 | std: 60.18171889277414, 26 | upper: 526, 27 | lower: 0, 28 | count: 75, 29 | count_ps: 7.5, 30 | sum: 652, 31 | mean: 8.693333333333333, 32 | median: 1 33 | } 34 | }, 35 | counter_rates: { 36 | 'statsd.bad_lines_seen': 0, 37 | 'statsd.packets_received': 15, 38 | 'central.api_requests': 7.5 39 | }, 40 | sets: { 41 | 'central.unique': [ 'value' ] 42 | }, 43 | pctThreshold: [ 90 ] 44 | }; 45 | -------------------------------------------------------------------------------- /tests/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var backend = require('../lib/'), 4 | events = require('events'), 5 | flush = require('./exampleFlush.js'), 6 | assert = require('chai').assert; 7 | 8 | var eventer = new events.EventEmitter(), 9 | config = { 10 | mongoHost: '127.0.0.1', 11 | flushInterval: 5000, 12 | debug: true 13 | }, 14 | backendRef; 15 | 16 | describe('the mongodb statsd backend', function(){ 17 | 18 | describe('verifying some deps', function(){ 19 | it('check existance of stubbed dependencies', function(d){ 20 | assert.ok(eventer); 21 | assert.ok(flush); 22 | d(); 23 | }); 24 | }); 25 | 26 | describe('backend#init()', function(){ 27 | it('initializes the backend', function(d){ 28 | backendRef = backend.init(new Date().getTime(), config, eventer); 29 | assert.ok(backendRef); 30 | d(); 31 | }); 32 | }); 33 | 34 | describe('eventer#emit("flush")', function(){ 35 | it('emits a flush event to the backend', function(d){ 36 | eventer.emit('flush', new Date().getTime(), flush); 37 | setTimeout(function(){ 38 | assert.ok(true); 39 | d(); 40 | }, 1500); 41 | }); 42 | }); 43 | }); 44 | --------------------------------------------------------------------------------