├── .checkbuild ├── .clever.json ├── .editorconfig ├── .gitignore ├── .jsbeautifyrc ├── .jshintrc ├── .nvmrc ├── README.md ├── api ├── api.js ├── api.test.js ├── controllers │ ├── index.js │ ├── measurements.js │ └── templates.js ├── domain │ ├── DateRangeInterval.ValueObject.js │ ├── DateRangeInterval.ValueObject.schema.js │ ├── DateRangeInterval.ValueObject.test.js │ ├── Measurement.Entity.js │ ├── Measurement.Entity.schema.js │ ├── Measurement.Entity.test.js │ ├── Measurement.Repository.js │ ├── Measurement.Repository.test.fixture.js │ ├── Measurement.Repository.test.js │ ├── MeasurementQuery.ValueObject.js │ ├── MeasurementQuery.ValueObject.schema.js │ ├── MeasurementQuery.test.fixture.js │ ├── MeasurementQuery.test.js │ ├── Template.ValueObject.js │ └── index.js ├── index.js ├── middlewares │ └── augmentReqAndRes.js └── routes.js ├── benchmark ├── data.json └── run.js ├── bootstrap.js ├── bootstrap.test.js ├── ci ├── .env_test ├── .gitkeep └── ci-start.sh ├── config ├── config.js └── index.js ├── docs ├── Elasticsearch.md └── es_post_example.js ├── helpers ├── PrettyError.js ├── amqp.js ├── elasticsearch.js ├── logger.js └── validation.js ├── package.json ├── server.js └── template ├── defined └── redsmin.template.js ├── index.js └── monitoring.template.js /.checkbuild: -------------------------------------------------------------------------------- 1 | { 2 | "checkbuild": { 3 | "enable": ["jshint", "jsinspect", "david", "nsp"], 4 | // don't exit immediately if one of the tools reports an error 5 | "continueOnError": true, 6 | // don't exit(1) even if we had some failures 7 | "allowFailures": false 8 | }, 9 | "jshint": { 10 | "args": ["**/*.js", "!*node_modules/**", "!benchmark/*"] 11 | }, 12 | "jscs": { 13 | "args": ["**/*.js", "!*node_modules/**", "!benchmark/*"] 14 | }, 15 | "jsinspect": { 16 | "args": ["**/*.js", "!*node_modules/**", "!benchmark/*", "!**/**.test.js"], 17 | "diff": true, 18 | "threshold": 40 19 | }, 20 | "buddyjs": { 21 | "args": ["**/*.js", "!*node_modules/**", "!benchmark/*"], 22 | "ignore": [0, 1, 200] 23 | }, 24 | "david": { 25 | "stable": true 26 | }, 27 | "nsp": {} 28 | } 29 | -------------------------------------------------------------------------------- /.clever.json: -------------------------------------------------------------------------------- 1 | {"apps":[{"app_id":"app_7afc88c7-9643-4607-8187-6977f60961e4","deploy_url":"git+ssh://git@push.par.clever-cloud.com/app_7afc88c7-9643-4607-8187-6977f60961e4.git","name":"5-statwarn-monitoring-api","alias":"5-statwarn-monitoring-api","org_id":"orga_212fb52d-b17d-43e8-bca7-6e69c0ce6911"}]} -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | 11 | # 4 space indentation 12 | [*.js] 13 | indent_style = space 14 | indent_size = 2 15 | 16 | 17 | # Indentation override for all JS under lib directory 18 | #[lib/**.js] 19 | #indent_style = space 20 | #indent_size = 2 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | node_modules 3 | .DS_Store 4 | npm-debug.log 5 | statwarn.png 6 | ci/.env_test 7 | benchmark/ 8 | -------------------------------------------------------------------------------- /.jsbeautifyrc: -------------------------------------------------------------------------------- 1 | { 2 | "indent_with_tabs": false, 3 | "max_preserve_newlines": 4, 4 | "preserve_newlines": true, 5 | "space_in_paren": false, 6 | "jslint_happy": true, 7 | "brace_style": "collapse", 8 | "keep_array_indentation": true, 9 | "keep_function_indentation": true, 10 | "eval_code": false, 11 | "unescape_strings": false, 12 | "break_chained_methods": false, 13 | "e4x": false, 14 | "wrap_line_length": 0, 15 | "format_on_save":true 16 | } 17 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "maxerr" : 50, // {int} Maximum error before stopping 3 | 4 | // Enforcing 5 | "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.) 6 | "camelcase" : false, // true: Identifiers must be in camelCase 7 | "curly" : true, // true: Require {} for every new block or scope 8 | "eqeqeq" : true, // true: Require triple equals (===) for comparison 9 | "forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty() 10 | "immed" : false, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());` 11 | "indent" : 2, // {int} Number of spaces to use for indentation 12 | "latedef" : false, // true: Require variables/functions to be defined before being used 13 | "newcap" : false, // true: Require capitalization of all constructor functions e.g. `new F()` 14 | "noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee` 15 | "noempty" : true, // true: Prohibit use of empty blocks 16 | "nonew" : false, // true: Prohibit use of constructors for side-effects (without assignment) 17 | "plusplus" : false, // true: Prohibit use of `++` & `--` 18 | "quotmark" : false, // Quotation mark consistency: 19 | // false : do nothing (default) 20 | // true : ensure whatever is used is consistent 21 | // "single" : require single quotes 22 | // "double" : require double quotes 23 | "undef" : true, // true: Require all non-global variables to be declared (prevents global leaks) 24 | "unused" : true, // true: Require all defined variables be used 25 | "strict" : true, // true: Requires all functions run in ES5 Strict Mode 26 | "trailing" : false, // true: Prohibit trailing whitespaces 27 | "maxparams" : 6, // {int} Max number of formal params allowed per function 28 | "maxdepth" : 4, // {int} Max depth of nested blocks (within functions) 29 | "maxstatements" : 25, // {int} Max number statements per function 30 | "maxcomplexity" : 6, // {int} Max cyclomatic complexity per function 31 | "maxlen" : false, // {int} Max number of characters per line 32 | 33 | // Relaxing 34 | "asi" : false, // true: Tolerate Automatic Semicolon Insertion (no semicolons) 35 | "boss" : false, // true: Tolerate assignments where comparisons would be expected 36 | "debug" : false, // true: Allow debugger statements e.g. browser breakpoints. 37 | "eqnull" : false, // true: Tolerate use of `== null` 38 | "es5" : false, // true: Allow ES5 syntax (ex: getters and setters) 39 | "esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`) 40 | "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features) 41 | // (ex: `for each`, multiple try/catch, function expression…) 42 | "evil" : false, // true: Tolerate use of `eval` and `new Function()` 43 | "expr" : false, // true: Tolerate `ExpressionStatement` as Programs 44 | "funcscope" : false, // true: Tolerate defining variables inside control statements" 45 | "globalstrict" : true, // true: Allow global "use strict" (also enables 'strict') 46 | "iterator" : false, // true: Tolerate using the `__iterator__` property 47 | "lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block 48 | "laxbreak" : false, // true: Tolerate possibly unsafe line breakings 49 | "laxcomma" : true, // true: Tolerate comma-first style coding 50 | "loopfunc" : false, // true: Tolerate functions being defined in loops 51 | "multistr" : true, // true: Tolerate multi-line strings 52 | "proto" : false, // true: Tolerate using the `__proto__` property 53 | "scripturl" : false, // true: Tolerate script-targeted URLs 54 | "smarttabs" : false, // true: Tolerate mixed tabs/spaces when used for alignment 55 | "shadow" : false, // true: Allows re-define variables later in code e.g. `var x=1; x=2;` 56 | "sub" : false, // true: Tolerate using `[]` notation when it can still be expressed in dot notation 57 | "supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;` 58 | "validthis" : false, // true: Tolerate using this in a non-constructor function 59 | 60 | // Environments 61 | "browser" : true, // Web Browser (window, document, etc) 62 | "couch" : false, // CouchDB 63 | "devel" : true, // Development/debugging (alert, confirm, etc) 64 | "dojo" : false, // Dojo Toolkit 65 | "jquery" : false, // jQuery 66 | "mootools" : false, // MooTools 67 | "node" : true, // Node.js 68 | "nonstandard" : false, // Widely adopted globals (escape, unescape, etc) 69 | "prototypejs" : false, // Prototype and Scriptaculous 70 | "rhino" : false, // Rhino 71 | "worker" : false, // Web Workers 72 | "wsh" : false, // Windows Scripting Host 73 | "yui" : false, // Yahoo User Interface 74 | 75 | // Legacy 76 | "nomen" : false, // true: Prohibit dangling `_` in variables 77 | "onevar" : false, // true: Allow only one `var` statement per function 78 | "passfail" : false, // true: Stop on first error 79 | "white" : false, // true: Check against strict whitespace and indentation rules 80 | 81 | // Custom Globals 82 | "predef" : [ 83 | "Buffer", 84 | "view", 85 | "test", 86 | "factoryKey", 87 | "RedisStub", 88 | "equal", 89 | "strictEqual", 90 | "ok", 91 | "deepEqual", 92 | "expect", 93 | "request", 94 | "_", 95 | "t", 96 | "async", 97 | "assert", 98 | "PrettyError", 99 | "checkGrid", 100 | 101 | "describe", 102 | "beforeEach", 103 | "afterEach", 104 | "before", 105 | "after", 106 | "it"] // additional predefined global variables 107 | } 108 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v0.10.35 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Setup 2 | 3 | ``` 4 | npm install 5 | ``` 6 | 7 | # Setup index template and document mapping 8 | 9 | ``` 10 | # edit `ELASTICSEARCH_INDEX_TEMPLATE` configuration parameter 11 | # setup template 12 | curl -XGET localhost:9000/internal/templates/setup 13 | ``` 14 | 15 | # API 16 | 17 | __baseUrl: monitoring.statwarn.com/api/v1__ 18 | 19 | ### POST /measurements/:id 20 | 21 | #### request 22 | 23 | __param:__ 24 | 25 | ``` 26 | timestamp: Number (optional) (format ISO 8601) 27 | ``` 28 | 29 | (json): 30 | 31 | ``` 32 | { 33 | id:"server-thor", 34 | timestamp: UTC, 35 | data:{ 36 | ram: 100000, // bits 37 | cpu: 10, // % 38 | network_io: 0 // bits 39 | }, 40 | metadata:{ 41 | 42 | } 43 | } 44 | ``` 45 | 46 | #### response 47 | 48 | 200 (OK) 49 | 50 | #### Errors 51 | ``` 52 | Incomplete 202: request accepted but the processing hasn't been completed 53 | Bad Request 400: missing field 54 | Internal Error 500: Internal server error 55 | Gateway Timeout 504: timeout 56 | ``` 57 | 58 | ### GET /measurements 59 | 60 | List all user's measurements according to following params 61 | 62 | __params:__ 63 | 64 | ``` 65 | field: String (e.g. 'data.a') 66 | fields: Strings (e.g. ['data.a', 'data.b']) 67 | id: String (e.g '17954235-926d-47af-8547-8b094556dbd6') 68 | ids: Strings (e.g. ['17954235-926d-47af-8547-8b094556dbd6', '33436bc1-5d55-44d8-9cb1-19d17823668c']) 69 | start_ts: Number (UTC) 70 | end_ts: Number (UTC) 71 | interval: String (e.g. year, quarter, month, week, day, hour, minute, second) 72 | agg: String (e.g. sum, avg, min, max, count (default. avg)) 73 | aggs: Strings (e.g. ['sum', 'avg', 'min']) 74 | ``` 75 | 76 | __response__ 77 | 78 | return the metric in json format with timestamp (example of metric_name: instantaneous_ops_per_sec) 79 | 80 | ``` 81 | [ 82 | { 83 | "id": String 84 | "field": String (name of the field), 85 | "values": [ 86 | { 87 | "timestamp": Number (UTC timestamp), 88 | "value": String, 89 | // and even more fields if agg=stats 90 | } 91 | ] 92 | }, 93 | 94 | ... 95 | ] 96 | ``` 97 | 98 | #### Errors 99 | 100 | ``` 101 | Bad Request 400: missing field 102 | Internal Error 500: Internal Server Error 103 | Gateway Timeout 504: timeout 104 | ``` 105 | 106 | ### GET /measurements/:id/describe 107 | 108 | example of metric : instantaneous_ops_per_sec 109 | 110 | __params:__ 111 | 112 | ``` 113 | size: Number (default 10) 114 | ``` 115 | 116 | __response__ 117 | 118 | return the format of the last `size` measurements 119 | 120 | ``` 121 | { 122 | instantaneous_ops_per_sec: "number" 123 | } 124 | ``` 125 | 126 | #### Errors 127 | 128 | ``` 129 | Internal Error 500: Internal Server Error 130 | Gateway Timeout 504: timeout 131 | ``` 132 | 133 | ### DEL /measurements 134 | 135 | Remove every metrics related to on or more `server_ids`. This route will be called each time a user remove its account or its own server. 136 | 137 | __params:__ 138 | 139 | ``` 140 | server_ids: String (e.g. 1,2,3) 141 | access_token: String 142 | ``` 143 | 144 | #### Errors 145 | 146 | ``` 147 | Incomplete 202: request accepted but the processing hasn't been completed 148 | Bad Request 400: missing field 149 | Internal Error 500: Internal server error 150 | Gateway Timeout 504: timeout 151 | ``` 152 | 153 | ### DEL /internal/removeAllData 154 | 155 | Remove datas older than `remove_before_date`. 156 | 157 | __params:__ 158 | 159 | ``` 160 | root_token: String 161 | remove\_before_date : Timestamp 162 | ``` 163 | -------------------------------------------------------------------------------- /api/api.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var express = require('express'); 4 | var bodyParser = require('body-parser'); 5 | 6 | module.exports = function (config, logger, es, amqp, template, fOnError) { 7 | var domain = require('./domain')(es, amqp, config, template); 8 | var routes = require('./routes')(logger, es, amqp, fOnError, domain); 9 | 10 | var app = express(); 11 | 12 | app.use(bodyParser.json()); 13 | app.use(bodyParser.urlencoded({ 14 | extended: true 15 | })); 16 | 17 | // specify routes 18 | routes(app); 19 | 20 | return app; 21 | }; 22 | -------------------------------------------------------------------------------- /api/api.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | // https://github.com/tj/supertest/blob/master/test/supertest.js 3 | require('../bootstrap.test'); 4 | var app, es, amqp; 5 | 6 | describe('Monitoring API server', function () { 7 | beforeEach(function (done) { 8 | t.getAPP(function (_app, config, logger, _es, _amqp) { 9 | app = _app; 10 | es = _es; 11 | amqp = _amqp; 12 | done(); 13 | }); 14 | }); 15 | 16 | describe('POST /api/v1/measurements', function () { 17 | it('should return a 400 error if nothing was passed in body', function (done) { 18 | request(app) 19 | .post('/api/v1/measurements/my-time-serie') 20 | .expect(400) 21 | .end(done); 22 | }); 23 | 24 | it('should return a 201 when a valid measurements is passed in body', function (done) { 25 | request(app) 26 | .post('/api/v1/measurements/my-time-serie') 27 | .send({ 28 | timestamp: Date.now(), 29 | data: { 30 | a: Math.round(Math.random() * 10), 31 | 'float': 1.234, 32 | 'floats': "1.234", 33 | b: 'plop' 34 | } 35 | }) 36 | .expect(201) 37 | .end(done); 38 | }); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /api/controllers/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function (models) { 4 | return { 5 | measurements: require('./measurements')(models), 6 | templates: require('./templates')(models) 7 | }; 8 | }; 9 | -------------------------------------------------------------------------------- /api/controllers/measurements.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function (domain) { 4 | return { 5 | // create a measurement 6 | post: function (req, res) { 7 | var measurement_id = req.params.measurement_id; 8 | var measurement = domain.Measurement.fromJSON(measurement_id, req.body); 9 | 10 | if (measurement instanceof PrettyError) { 11 | return res.error(measurement); 12 | } 13 | 14 | domain.Measurements.create(measurement, function (err) { 15 | if (err) { 16 | return res.error(err); 17 | } 18 | res.status(201).end(); 19 | }); 20 | }, 21 | 22 | // retrieve one (or more) measurements from a id 23 | get: function (req, res) { 24 | var dateRangeInterval = domain.DateRangeInterval.fromReq(req); 25 | if (dateRangeInterval instanceof PrettyError) { 26 | return res.error(dateRangeInterval); 27 | } 28 | 29 | var measurementQuery = domain.MeasurementQuery.fromReq(req, dateRangeInterval); 30 | if (measurementQuery instanceof PrettyError) { 31 | return res.error(measurementQuery); 32 | } 33 | 34 | domain.Measurements.findByIds(measurementQuery, function (err, data, took) { 35 | if (err) { 36 | return res.error(err); 37 | } 38 | 39 | res.set('x-took', took); 40 | res.ok(data); 41 | }); 42 | }, 43 | 44 | // get data keys of X latest documents 45 | describe: function (req, res) { 46 | if (!_.isObject(req) || !req || !_.isObject(req.params)) { 47 | return res.error(new PrettyError(400, 'Invalid request')); 48 | } 49 | 50 | if (size && !_.isNumber(parseInt(size, 10))) { 51 | return res.error(new PrettyError(400, 'size must be a number')); 52 | } 53 | 54 | var id = req.params.measurement_id; 55 | var size = size ? parseInt(req.params.size, 10) :  10; 56 | 57 | domain.Measurements.describe(id, size, function (err, data) { 58 | if (err) { 59 | return res.error(err); 60 | } 61 | res.ok(data); 62 | }); 63 | }, 64 | 65 | removeAllData: function (req, res) { 66 | if (!_.isObject(req) || !req || !_.isObject(req.params)) { 67 | return res.error(new PrettyError(400, 'Invalid request')); 68 | } 69 | 70 | var before_date = req.params.remove_before_date; 71 | 72 | if (!before_date || !_.isNumber(parseInt(before_date, 10))) { 73 | return res.error(new PrettyError(400, 'before_date must be defined and a timestamp')); 74 | } 75 | 76 | domain.Measurements.removeAllData(before_date, function (err) { 77 | if (err) { 78 | return res.error(err); 79 | } 80 | res.status(201).end(); 81 | }); 82 | }, 83 | 84 | // Define middlewares 85 | middlewares: [ 86 | 87 | function checkMeasurementId(req, res, next) { 88 | if (!req.params || !req.params.measurement_id || !_.isString(req.params.measurement_id) || req.params.measurement_id.length === 0) { 89 | return res.error(new PrettyError(400, 'measurement_id must be defined and a non-empty string')); 90 | } 91 | 92 | next(); 93 | }] 94 | }; 95 | }; 96 | -------------------------------------------------------------------------------- /api/controllers/templates.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function (domain) { 4 | return { 5 | // setup a template 6 | setup: function (req, res) { 7 | domain.Template.setupTemplate(res.errorOrValue); 8 | } 9 | }; 10 | }; 11 | -------------------------------------------------------------------------------- /api/domain/DateRangeInterval.ValueObject.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var moment = require('moment-range'); 3 | /** 4 | * DateRangeInterval value 5 | */ 6 | 7 | module.exports = function () { 8 | /** 9 | * [DateRangeInterval description] 10 | * @param {Number} start_ts 11 | * @param {Number} end_ts 12 | * @param {String} interval 13 | */ 14 | function DateRangeInterval(start_ts, end_ts, interval) { 15 | this.start_ts = start_ts; 16 | this.end_ts = end_ts; 17 | this.interval = interval; 18 | 19 | this._range = moment.range(start_ts, end_ts); 20 | this._interval = interval; 21 | } 22 | 23 | DateRangeInterval.schema = require('./DateRangeInterval.ValueObject.schema'); 24 | 25 | /** 26 | * Create a new DateRangeInterval object from a query 27 | * This factory will return a PrettyError if the data are invalid 28 | * @param {Express req} req 29 | * req.start_ts (string) e.g. '1419872302441' 30 | * req.end_ts (string) e.g. '1419872392441' 31 | * req.interval must be a string from the `DateRangeInterval.INTERVALS` interval set 32 | * @return {PrettyError|DateRangeInterval} 33 | */ 34 | DateRangeInterval.fromReq = function (req) { 35 | if (!_.isObject(req) || !req || !_.isObject(req.query)) { 36 | return new PrettyError(400, 'Invalid request'); 37 | } 38 | 39 | req.query.start_ts = parseInt(req.query.start_ts, 10); 40 | req.query.end_ts = parseInt(req.query.end_ts, 10); 41 | 42 | return _.validate(req.query, DateRangeInterval.schema.req, function fallback(query) { 43 | return new DateRangeInterval(query.start_ts, query.end_ts, query.interval); 44 | }); 45 | }; 46 | 47 | return DateRangeInterval; 48 | }; 49 | -------------------------------------------------------------------------------- /api/domain/DateRangeInterval.ValueObject.schema.js: -------------------------------------------------------------------------------- 1 | // http://json-schema.org/ 2 | module.exports = { 3 | req: { 4 | 'title': 'fromReq Schema', 5 | 'type': 'object', 6 | 7 | 'properties': { 8 | 'interval': { 9 | 'title': 'Metric interval', 10 | 'description': 'Interval', 11 | 'type': 'string', 12 | 'format': 'enum', 13 | 'values': ['year', 'quarter', 'month', 'week', 'day', 'hour', 'minute', 'second'] 14 | }, 15 | 'start_ts': { 16 | 'title': 'date range start date', 17 | 'description': '', 18 | 'type': 'number' 19 | }, 20 | 'end_ts': { 21 | 'title': 'date range end date', 22 | 'description': '', 23 | 'type': 'number' 24 | } 25 | }, 26 | 'required': ['interval', 'start_ts', 'end_ts'] 27 | } 28 | }; 29 | -------------------------------------------------------------------------------- /api/domain/DateRangeInterval.ValueObject.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | require('../../bootstrap.test'); 3 | 4 | var DateRangeInterval = require('./DateRangeInterval.ValueObject')(); 5 | 6 | describe('DateRangeInterval', function () { 7 | describe('DateRangeInterval.fromReq', function () { 8 | it('should return an error if the query is not defined', function (done) { 9 | t.isPrettyError(DateRangeInterval.fromReq(), 400); 10 | done(); 11 | }); 12 | 13 | it('should return an error if the interval is not defined', function (done) { 14 | t.ok(DateRangeInterval.fromReq({ 15 | query: {} 16 | }) instanceof PrettyError, 'should be a pretty error'); 17 | // @todo check error.details 18 | done(); 19 | }); 20 | 21 | it('should return an error if the interval is defined but invalid', function (done) { 22 | t.ok(DateRangeInterval.fromReq({ 23 | query: { 24 | interval: 'plop' 25 | } 26 | }) instanceof PrettyError, 'should be a pretty error'); 27 | // @todo check error.details 28 | done(); 29 | }); 30 | 31 | it('should return an interval if the interval is valid', function (done) { 32 | var dateRangeInterval = DateRangeInterval.fromReq({ 33 | query: { 34 | interval: 'minute', 35 | start_ts: String(+new Date() - 3600 * 60 * 1000), // -1 hour 36 | end_ts: String(+new Date()), // we convert *_ts to string because it will always be sent as string 37 | } 38 | }); 39 | 40 | if (dateRangeInterval instanceof PrettyError) { 41 | console.error(dateRangeInterval); 42 | } 43 | 44 | t.ok(dateRangeInterval instanceof DateRangeInterval, 'should be an instance of interval is valid'); 45 | done(); 46 | }); 47 | }); 48 | }); 49 | -------------------------------------------------------------------------------- /api/domain/Measurement.Entity.js: -------------------------------------------------------------------------------- 1 | /*jshint -W030 */ 2 | 'use strict'; 3 | 4 | /** 5 | * Measurement entity 6 | * @param {String} id 7 | * time serie id 8 | * @param {Number} timestamp 9 | * timestamp in UTC 10 | * @param {Object} data pair of key/values 11 | * @param {Object|Null} metadata pair of key/values 12 | */ 13 | function Measurement(id, timestamp, data, metadata) { 14 | assert(_.isString(id)); 15 | assert(_.isNumber(timestamp)); 16 | assert(_.isPlainObject(data)); 17 | metadata && assert(_.isPlainObject(metadata)); 18 | !metadata && assert(_.isNull(metadata)); 19 | 20 | this.id = id; 21 | this.timestamp = timestamp; 22 | this.data = data; 23 | this.metadata = metadata; 24 | } 25 | 26 | Measurement.schema = require('./Measurement.Entity.schema'); 27 | 28 | /** 29 | * Create a new Measurement object from a JSON 30 | * This factory will return a PrettyError if data are invalid 31 | * @param {Object} 32 | * @return {PrettyError|Measurement} 33 | */ 34 | Measurement.fromJSON = function (measurement_id, json) { 35 | if (!_.isObject(json) || !json) { 36 | return new PrettyError(400, 'Invalid JSON for Measurement'); 37 | } 38 | 39 | json.id = measurement_id; 40 | return _.validate(json, Measurement.schema, function fallback(json) { 41 | return new Measurement(json.id, json.timestamp, json.data, json.metadata || null); 42 | }); 43 | }; 44 | 45 | /** 46 | * Convert the current measurement to a Document 47 | * @return {Object} JSON object 48 | */ 49 | Measurement.prototype.toDocument = function () { 50 | var omit = ['id']; 51 | if (!this.metadata) { 52 | omit.push('metadata'); 53 | } 54 | 55 | return _.omit(this, omit); 56 | }; 57 | 58 | module.exports = Measurement; 59 | -------------------------------------------------------------------------------- /api/domain/Measurement.Entity.schema.js: -------------------------------------------------------------------------------- 1 | // http://json-schema.org/ 2 | module.exports = { 3 | 'title': 'Measurement schema', 4 | 'type': 'object', 5 | 'properties': { 6 | 'id': { 7 | 'title': 'Measurement id', 8 | 'description': 'The id the current measurement data should be linked to', 9 | 'type': 'string', 10 | 'minLength': 3, 11 | 'maxLength': 125 12 | }, 13 | 14 | 'timestamp': { 15 | 'title': 'Measurement timestamp', 16 | 'description': 'Timestamp UTC', 17 | 'type': 'number', 18 | 'minimum': 0 19 | }, 20 | 21 | 'data': { 22 | 'title': 'Measurement data', 23 | 'description': '', 24 | 'type': 'object', 25 | 'format': 'single-level-object' 26 | }, 27 | 28 | 'metadata': { 29 | 'title': 'Measurement metadata', 30 | 'description': '', 31 | 'type': 'object' 32 | } 33 | }, 34 | 'required': ['id', 'timestamp', 'data'] 35 | }; 36 | -------------------------------------------------------------------------------- /api/domain/Measurement.Entity.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | require('../../bootstrap.test'); 3 | 4 | var Measurement = require('./Measurement.Entity'); 5 | 6 | describe('Measurement', function () { 7 | describe('Measurement.fromJSON', function () { 8 | it('should return an error if we try to pass it a nested object', function () { 9 | var err = Measurement.fromJSON('random-id', { 10 | timestamp: 1388250221005, 11 | data: { 12 | uptime_in_seconds: 1588578, 13 | connected: 1, 14 | commands: { 15 | plop: 1 16 | } 17 | } 18 | }); 19 | 20 | t.isPrettyError(err, 400); 21 | }); 22 | 23 | it('should return an error if the measurement id is too long', function () { 24 | var MEASUREMENT_ID = _.repeat('-', 150); 25 | 26 | var err = Measurement.fromJSON(MEASUREMENT_ID, { 27 | timestamp: 1388250221005, 28 | data: { 29 | uptime_in_seconds: 1588578, 30 | connected: 1 31 | } 32 | }); 33 | 34 | t.isPrettyError(err, 400); 35 | }); 36 | 37 | it('should return an error if the measurement id is too short', function () { 38 | var MEASUREMENT_ID = _.repeat('-', 1); 39 | 40 | var err = Measurement.fromJSON(MEASUREMENT_ID, { 41 | timestamp: 1388250221005, 42 | data: { 43 | uptime_in_seconds: 1588578, 44 | connected: 1 45 | } 46 | }); 47 | 48 | t.isPrettyError(err, 400); 49 | }); 50 | 51 | it('should return an valid measurement', function () { 52 | var MEASUREMENT_ID = 'this-is-a-test'; 53 | 54 | var measurement = Measurement.fromJSON(MEASUREMENT_ID, { 55 | timestamp: 1388250221005, 56 | data: { 57 | uptime_in_seconds: 1588578, 58 | connected: 1 59 | } 60 | }); 61 | 62 | t.instanceOf(measurement, Measurement); 63 | }); 64 | }); 65 | }); 66 | -------------------------------------------------------------------------------- /api/domain/Measurement.Repository.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // MeasurementRepository 4 | module.exports = function (es, amqp, config, DateRangeInterval, MeasurementQuery) { 5 | assert(_.isString(config.elasticsearch.index.name_prefix)); 6 | assert(_.isString(config.elasticsearch.index.document.type)); 7 | 8 | assert(_.isString(config.amqp.publish.publish_key)); 9 | assert(_.isString(config.amqp.publish.exchange)); 10 | 11 | assert(_.isString(config.statwarn.schema.monitoring.create)); 12 | 13 | assert(amqp.publishExchange); 14 | 15 | var INDEX_NAME_PREFIX = config.elasticsearch.index.name_prefix; 16 | var INDEX_DOCUMENT_TYPE = config.elasticsearch.index.document.type; 17 | 18 | var MONITORING_PUBLISH_KEY = config.amqp.publish.publish_key; 19 | var MONITORING_EXCHANGE = config.amqp.publish.exchange; 20 | var SCHEMA_MONITORING_CREATE = config.statwarn.schema.monitoring.create; 21 | 22 | var Measurement = require('./Measurement.Entity'); 23 | 24 | function makeIndexFromId(id) { 25 | return INDEX_NAME_PREFIX + '-' + id; 26 | } 27 | 28 | function makePublishKeyFromId(id) { 29 | return MONITORING_PUBLISH_KEY + '-' + id; 30 | } 31 | 32 | function publishOnRabbitMQ(measurement, f) { 33 | var message = createEnvelopeFromMeasurement(measurement); 34 | var publish_key = makePublishKeyFromId(measurement.id); 35 | // then publish it in AMQP 36 | amqp.publishExchange.publish(publish_key, message, { 37 | mandatory: true, 38 | confirm: true, 39 | exchange: MONITORING_EXCHANGE, 40 | type: SCHEMA_MONITORING_CREATE 41 | }, function onConfirm() { 42 | f(null); 43 | }); 44 | } 45 | 46 | function createEnvelopeFromMeasurement(measurement) { 47 | return { 48 | id: 'evt_' + (+new Date()), // @todo create a UUID instead 49 | created: +new Date(), // current date 50 | type: SCHEMA_MONITORING_CREATE, 51 | data: measurement 52 | }; 53 | } 54 | 55 | return { 56 | /** 57 | * Create a measurement inside storage backend 58 | * @param {Measurement} metric 59 | * @param {Function} f(PrettyError) 60 | */ 61 | create: function (measurement, f) { 62 | assert(measurement instanceof Measurement); 63 | 64 | // first write the measurement in ES 65 | es.create({ 66 | index: makeIndexFromId(measurement.id), 67 | type: INDEX_DOCUMENT_TYPE, 68 | body: measurement.toDocument() 69 | }, function (err, res) { 70 | if (err) { 71 | return f(new PrettyError(500, 'An error occured while creating the measurement', err)); 72 | } 73 | 74 | if (!_.isPlainObject(res) || !res.created) { 75 | return f(new PrettyError(500, 'Could not create the measurement', err)); 76 | } 77 | 78 | publishOnRabbitMQ(measurement, f); 79 | }); 80 | }, 81 | 82 | /** 83 | * [findByIds description] 84 | * @param {Array} ids array of time serie ids 85 | * @param {Array} fields 86 | * @param {Array} aggs array of aggregation type 87 | * @param {DateRangeInterval} dateRangeInterval 88 | * @param {Function} f(err: PrettyError, data: Array, took: Number) 89 | */ 90 | findByIds: function (measurementQuery, f) { 91 | assert(measurementQuery instanceof MeasurementQuery); 92 | 93 | console.log(JSON.stringify(measurementQuery.buildQuery(makeIndexFromId, INDEX_DOCUMENT_TYPE), null, 2)); 94 | es.search(measurementQuery.buildQuery(makeIndexFromId, INDEX_DOCUMENT_TYPE), function (err, result) { 95 | if (err) { 96 | return f(new PrettyError(500, 'Could not retrieve measurement, try again.', err)); 97 | } 98 | 99 | return f(null, measurementQuery.parseResults(result), result.took); 100 | }); 101 | }, 102 | 103 | describe: function (id, size, f) { 104 | es.search({ 105 | index: makeIndexFromId(id), 106 | type: INDEX_DOCUMENT_TYPE, 107 | body: { 108 | size: size, 109 | sort: [{ 110 | timestamp: { 111 | order: 'desc' 112 | } 113 | }] 114 | }, 115 | }, function (err, res) { 116 | if (err || !res || !res.hits.hits) { 117 | return f(new PrettyError(500, 'Could not retrieve measurement, try again.', err)); 118 | } 119 | // only get source.data of document 120 | var source = _(res.hits.hits) 121 | .pluck('_source') 122 | .pluck('data') 123 | .value(); 124 | 125 | // merge measurements and replace value by key type 126 | var allKeys = _.mapValues(_.extend.apply(null, source), function (v) { 127 | // typeof [] === 'object' 128 | return _.isArray(v) ? 'array' : typeof v; 129 | }); 130 | 131 | return f(null, allKeys); 132 | }); 133 | }, 134 | 135 | removeAllData: function (before_date, f) { 136 | // get indices starting by 'monitoring' 137 | es.indices.getAliases({ 138 | index: INDEX_NAME_PREFIX + '*', 139 | type: INDEX_DOCUMENT_TYPE 140 | }, function (err, indices) { 141 | if (err) { 142 | return f(err); 143 | } 144 | 145 | // remove documents older than `before_date` for each index 146 | es.deleteByQuery({ 147 | // comma separated indices 148 | index: Object.keys(indices).join(','), 149 | body: { 150 | query: { 151 | range: { 152 | timestamp: { 153 | lte: before_date 154 | } 155 | } 156 | } 157 | } 158 | }, function (err) { 159 | if (err) { 160 | return f(err); 161 | } 162 | }); 163 | }); 164 | } 165 | }; 166 | }; 167 | -------------------------------------------------------------------------------- /api/domain/Measurement.Repository.test.fixture.js: -------------------------------------------------------------------------------- 1 | module.exports = [{ 2 | a: 1 3 | }, { 4 | b: 'plop' 5 | }, { 6 | c: true 7 | }]; 8 | -------------------------------------------------------------------------------- /api/domain/Measurement.Repository.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | require('../../bootstrap.test'); 3 | 4 | var logger = require('../../helpers/logger'); 5 | var config = require('../../config')(logger); 6 | var INDEX_NAME_PREFIX = config.elasticsearch.index.name_prefix; 7 | var INDEX_DOCUMENT_TYPE = config.elasticsearch.index.document.type; 8 | var connectAndCheckES = require('../../helpers/elasticsearch')(config.elasticsearch); 9 | var testObjCollection = require('./Measurement.Repository.test.fixture'); 10 | var Measurement; 11 | 12 | describe('Measurement Repository', function () { 13 | describe('Measurement.describe', function () { 14 | 15 | // insert tests documents into monitoring-test/measurement 16 | before(function (done) { 17 | connectAndCheckES(function (err, es) { 18 | Measurement = require('./Measurement.Repository')(es, { 19 | publishExchange: { 20 | publish: function (a, b, c, f) { 21 | f(); 22 | } 23 | } 24 | }, config); 25 | 26 | async.eachSeries(testObjCollection, function postDummyOnES(obj, cb) { 27 | var id = 'test'; 28 | var entity = require('./Measurement.Entity').fromJSON(id, { 29 | timestamp: _.now(), 30 | data: obj 31 | }); 32 | 33 | Measurement.create(entity, function (err) { 34 | if (err) { 35 | throw err; 36 | } 37 | cb(); 38 | }); 39 | }, function () { 40 | setTimeout(function () { 41 | done(); 42 | }, 2000); 43 | }); 44 | }); 45 | }); 46 | 47 | // remove those documents 48 | after(function (done) { 49 | connectAndCheckES(function (err, es) { 50 | es.deleteByQuery({ 51 | index: INDEX_NAME_PREFIX + '-test', 52 | type: INDEX_DOCUMENT_TYPE, 53 | body: { 54 | query: { 55 | match_all: {} 56 | } 57 | } 58 | }, function (err) { 59 | if (err) { 60 | throw err; 61 | } 62 | done(); 63 | }); 64 | }); 65 | }); 66 | 67 | it('should return a PrettyError', function (done) { 68 | var id = 'plop'; 69 | var size = 10; 70 | Measurement.describe(id, size, function (err) { 71 | t.isPrettyError(err, 500); 72 | done(); 73 | }); 74 | }); 75 | 76 | it('should return the whole collection because of size 10', function (done) { 77 | var id = 'test'; 78 | var size = 10; 79 | 80 | Measurement.describe(id, size, function (err, keys) { 81 | t.equal(err instanceof PrettyError, false); 82 | t.deepEqual(keys, { 83 | a: 'number', 84 | b: 'string', 85 | c: 'boolean' 86 | }); 87 | done(); 88 | }); 89 | }); 90 | 91 | it('should return the first two documents merged ', function (done) { 92 | var id = 'test'; 93 | var size = 2; 94 | Measurement.describe(id, size, function (err, keys) { 95 | t.equal(err instanceof PrettyError, false); 96 | t.deepEqual(keys, { 97 | b: 'string', 98 | c: 'boolean' 99 | }); 100 | done(); 101 | }); 102 | }); 103 | }); 104 | }); 105 | -------------------------------------------------------------------------------- /api/domain/MeasurementQuery.ValueObject.js: -------------------------------------------------------------------------------- 1 | /*jshint maxcomplexity: 17*/ 2 | 'use strict'; 3 | /** 4 | * MeasurementQuery value 5 | */ 6 | var filtrES = require('filtres'); 7 | 8 | module.exports = function (DateRangeInterval) { 9 | /** 10 | * Represent a measurement query 11 | * @param {Array} ids 12 | * @param {Array} fields 13 | * @param {Array} aggs 14 | * @param {DateRangeInterval} aggs 15 | * @private 16 | */ 17 | function MeasurementQuery(ids, fields, filters, aggs, dateRangeInterval) { 18 | this.ids = ids; 19 | this.fields = fields; 20 | this.filters = filters; 21 | this.aggs = aggs; 22 | this.range = dateRangeInterval; 23 | } 24 | 25 | MeasurementQuery.schema = require('./MeasurementQuery.ValueObject.schema'); 26 | 27 | MeasurementQuery.AGGS_MAPPING = { 28 | // public aggregation name -> private name 29 | 'min': 'min', 30 | 'max': 'max', 31 | 'sum': 'sum', 32 | 'avg': 'avg', 33 | 'count': 'value_count', 34 | 'stats': 'extended_stats' 35 | }; 36 | 37 | MeasurementQuery.DEFAULT_AGGREGATION_TYPE = _.first(MeasurementQuery.schema.AGGREGATION_TYPES); 38 | 39 | // ensure at start time that the schema and the mapping are kept in sync 40 | assert(_.intersection(MeasurementQuery.schema.AGGREGATION_TYPES, _.keys(MeasurementQuery.AGGS_MAPPING)).length === MeasurementQuery.schema.AGGREGATION_TYPES.length); 41 | 42 | /** 43 | * @param {Function} makeIndexFromId(id) -> String a delegate 44 | * @param {String} index_document_type document type 45 | * @return {Object} elasticsearch query 46 | */ 47 | MeasurementQuery.prototype.buildQuery = function (makeIndexFromId, index_document_type) { 48 | // First build the aggregation on fields 49 | var fields_aggs = this.fields.reduce(function (obj, field, i) { 50 | // ES needs an object of: 51 | // 52 | // 'used_memory': { 53 | // avg: { 54 | // field: 'used_memory' 55 | // } 56 | // } 57 | // 58 | // with "used_memory" == field 59 | 60 | obj[field] = {}; 61 | var public_aggregate_name = this.aggs[i]; 62 | var es_aggregate_type = MeasurementQuery.AGGS_MAPPING[public_aggregate_name]; 63 | obj[field][es_aggregate_type] = { 64 | field: field 65 | }; 66 | return obj; 67 | }.bind(this), {}); 68 | 69 | return { 70 | indices: this.ids.map(makeIndexFromId), 71 | type: index_document_type, 72 | fields: this.fields, 73 | search_type: 'count', 74 | body: { 75 | // size=0 to not show search hits because we only want to see the aggregation results in the response. 76 | size: 0, 77 | 78 | query: { 79 | filtered: { 80 | filter: { 81 | bool: { 82 | must: [{ 83 | range: { 84 | timestamp: { 85 | from: this.range.start_ts, 86 | to: this.range.end_ts 87 | } 88 | } 89 | }, 90 | this.filters 91 | ] 92 | } 93 | } 94 | } 95 | }, 96 | 97 | aggs: { 98 | volume: { 99 | // histogram aggregation: http://www.elasticsearch.com/guide/en/elasticsearch/reference/current/search-aggregations-bucket-histogram-aggregation.html 100 | // date histogram aggregation: http://www.elasticsearch.com/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html 101 | date_histogram: { 102 | field: 'timestamp', 103 | min_doc_count: 0, 104 | interval: this.range.interval, 105 | extended_bounds: { 106 | min: this.range.start_ts, 107 | max: this.range.end_ts 108 | } 109 | }, 110 | aggs: fields_aggs 111 | } 112 | } 113 | } 114 | }; 115 | }; 116 | 117 | /** 118 | * Parse the result from an ES query based on the MeasurementQuery 119 | * @param {Object} result 120 | * ES query result 121 | * @return {Array} array of buckets 122 | */ 123 | MeasurementQuery.prototype.parseResults = function (results) { 124 | // buckets is an array of 125 | // [{ 126 | // key_as_string: "2013-12-29T15:54:00.000Z", 127 | // key: 1388332440000, 128 | // doc_count: 1, 129 | // {{fields[0]}}: { <-- first field name 130 | // value: 3626992 131 | // }, 132 | // ... <-- and so on for other fields 133 | // }, 134 | // ...] 135 | // 136 | 137 | /** 138 | Output format must be: 139 | [ 140 | { 141 | "id": String 142 | "field": String (name of the field), 143 | "values": [ 144 | { 145 | "timestamp": Number (UTC timestamp), 146 | "value": String, 147 | // and even more fields if agg=stats 148 | } 149 | ] 150 | }, 151 | ... 152 | ] 153 | */ 154 | 155 | return results.aggregations.volume.buckets.reduce(function reduceBuckets(data, bucket) { 156 | this.fields.forEach(function forEachField(field) { 157 | // first find output data bucket 158 | var group = _.find(data, { 159 | field: field 160 | }); 161 | 162 | 163 | if (!group) { 164 | group = { 165 | id: this.ids[0], // @todo handle multiple ids 166 | field: field, 167 | values: [] 168 | }; 169 | data.push(group); 170 | } 171 | 172 | group.values.push(_.extend({ 173 | // `key` and rename it to `timestamp` 174 | timestamp: bucket.key 175 | }, 176 | // each fields along with their values 177 | bucket[field] 178 | )); 179 | }, this); 180 | 181 | return data; 182 | }.bind(this), []); 183 | }; 184 | 185 | /** 186 | * Create a new MeasurementQuery object from a query 187 | * This factory will return a PrettyError if the data are invalid 188 | * @param {Express req} req 189 | * 190 | * req.id (string) e.g. '1419872302441' 191 | * OR 192 | * req.ids (array) e.g. ['1419872302441'] 193 | * 194 | * req.field (string) e.g. 'used_memory' 195 | * OR 196 | * req.fields (array) e.g. ['used_memory', ..] 197 | * 198 | * req.agg (string) e.g. 'stats' 199 | * OR 200 | * req.aggs (array) e.g. ['stats', ' 201 | * @param {DateRangeInterval} dateRangeInterval 202 | * @return {PrettyError|MeasurementQuery} 203 | */ 204 | MeasurementQuery.fromReq = function (req, dateRangeInterval) { 205 | if (!_.isObject(req) || !req || !_.isObject(req.query)) { 206 | return new PrettyError(400, 'Invalid request'); 207 | } 208 | 209 | if (!(dateRangeInterval instanceof DateRangeInterval)) { 210 | return new PrettyError(400, 'Invalid Date range interval'); 211 | } 212 | 213 | if (!req.query.id && !req.query.ids) { 214 | return new PrettyError(400, 'id or ids must be defined'); 215 | } 216 | 217 | if (!req.query.field && !req.query.fields) { 218 | return new PrettyError(400, 'field or fields must be defined'); 219 | } 220 | 221 | req.query.ids = convertToArray(req.query.id || req.query.ids); 222 | req.query.fields = convertToArray(req.query.field || req.query.fields); 223 | req.query.aggs = convertToArray(req.query.agg || req.query.aggs); 224 | req.query.filters = req.query.filter ||  req.query.filters ||  ''; 225 | 226 | if (req.query.aggs.length > 0 && req.query.aggs.length !== req.query.fields.length) { 227 | return new PrettyError(400, 'For each `fields` specified an aggregation type (`aggs`) must be specified'); 228 | } 229 | 230 | if (req.query.aggs.length === 0) { 231 | // not aggs were specify fill it 232 | req.query.aggs = req.query.fields.map(_.partial(_.identity, MeasurementQuery.DEFAULT_AGGREGATION_TYPE)); 233 | } 234 | 235 | if (!_.isString(req.query.filters)) { 236 | return new PrettyError(400, 'filter or filters must be a string'); 237 | } 238 | 239 | try { 240 | req.query.filters = !_.isEmpty(req.query.filters) ? filtrES.compile(req.query.filters).query.filtered.filter : {}; 241 | } catch (err) { 242 | return new PrettyError(400, 'filter or filters are invalid', err); 243 | } 244 | 245 | return _.validate(req.query, MeasurementQuery.schema.fromReq, function fallback(query) { 246 | return new MeasurementQuery(query.ids, query.fields, query.filters, query.aggs, dateRangeInterval); 247 | }); 248 | }; 249 | 250 | return MeasurementQuery; 251 | }; 252 | 253 | ///////////// 254 | // Helpers // 255 | ///////////// 256 | 257 | function convertToArray(value) { 258 | if (_.isString(value)) { 259 | return [value]; 260 | } 261 | 262 | if (_.isArray(value)) { 263 | return value; 264 | } 265 | 266 | // invalid value 267 | return []; 268 | } 269 | -------------------------------------------------------------------------------- /api/domain/MeasurementQuery.ValueObject.schema.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | // http://json-schema.org/ 3 | 4 | var AGGREGATION_TYPES = ['avg', 'min', 'max', 'sum', 'count', 'stats']; 5 | 6 | module.exports = { 7 | AGGREGATION_TYPES: AGGREGATION_TYPES, 8 | fromReq: { 9 | 'title': 'fromReq Schema', 10 | 'type': 'object', 11 | 'properties': { 12 | 'ids': { 13 | 'title': 'metric id(s) to fetch', 14 | 'description': '', 15 | 'type': 'array', 16 | 'additionalItems': false, 17 | 'required': true, 18 | 'items': { 19 | 'type': ['string'] 20 | } 21 | }, 22 | 'fields': { 23 | 'title': 'field(s) to fetch', 24 | 'description': '', 25 | 'type': 'array', 26 | 'additionalItems': false, 27 | 'required': true, 28 | 'items': { 29 | 'type': ['string'] 30 | } 31 | }, 32 | 'aggs': { 33 | 'title': 'aggregations type', 34 | 'description': '', 35 | 'type': 'array', 36 | 'additionalItems': false, 37 | 'required': true, 38 | 'items': { 39 | 'type': 'string', 40 | 'format': 'enum', 41 | 'values': AGGREGATION_TYPES 42 | } 43 | } 44 | } 45 | } 46 | }; 47 | -------------------------------------------------------------------------------- /api/domain/MeasurementQuery.test.fixture.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | // ?aggs=stats&field=used_memory&interval=minute&start_ts=0... 3 | single_field_used_memory_agg_stats: { 4 | "took": 4, 5 | "timed_out": false, 6 | "_shards": { 7 | "total": 3, 8 | "successful": 3, 9 | "failed": 0 10 | }, 11 | "hits": { 12 | "total": 347, 13 | "max_score": 0, 14 | "hits": [] 15 | }, 16 | "aggregations": { 17 | "volume": { 18 | "buckets": [{ 19 | "key_as_string": "2013-12-29T15:54:00.000Z", 20 | "key": 1388332440000, 21 | "doc_count": 1, 22 | "used_memory": { 23 | "count": 1, 24 | "min": 3626992, 25 | "max": 3626992, 26 | "avg": 3626992, 27 | "sum": 3626992, 28 | "sum_of_squares": 13155070968064, 29 | "variance": 0, 30 | "std_deviation": 0 31 | } 32 | }, { 33 | "key_as_string": "2013-12-29T15:55:00.000Z", 34 | "key": 1388332500000, 35 | "doc_count": 1, 36 | "used_memory": { 37 | "count": 1, 38 | "min": 3177480, 39 | "max": 3177480, 40 | "avg": 3177480, 41 | "sum": 3177480, 42 | "sum_of_squares": 10096379150400, 43 | "variance": 0, 44 | "std_deviation": 0 45 | } 46 | }, { 47 | "key_as_string": "2013-12-29T15:56:00.000Z", 48 | "key": 1388332560000, 49 | "doc_count": 1, 50 | "used_memory": { 51 | "count": 1, 52 | "min": 1197328496, 53 | "max": 1197328496, 54 | "avg": 1197328496, 55 | "sum": 1197328496, 56 | "sum_of_squares": 1433595527333622000, 57 | "variance": 0, 58 | "std_deviation": 0 59 | } 60 | }, { 61 | "key_as_string": "2013-12-29T15:57:00.000Z", 62 | "key": 1388332620000, 63 | "doc_count": 1, 64 | "used_memory": { 65 | "count": 1, 66 | "min": 14033800, 67 | "max": 14033800, 68 | "avg": 14033800, 69 | "sum": 14033800, 70 | "sum_of_squares": 196947542440000, 71 | "variance": 0, 72 | "std_deviation": 0 73 | } 74 | }, { 75 | "key_as_string": "2013-12-29T15:58:00.000Z", 76 | "key": 1388332680000, 77 | "doc_count": 0, 78 | "used_memory": { 79 | "count": 0, 80 | "min": null, 81 | "max": null, 82 | "avg": null, 83 | "sum": null, 84 | "sum_of_squares": null, 85 | "variance": null, 86 | "std_deviation": null 87 | } 88 | }, { 89 | "key_as_string": "2013-12-29T15:59:00.000Z", 90 | "key": 1388332740000, 91 | "doc_count": 1, 92 | "used_memory": { 93 | "count": 1, 94 | "min": 40013264, 95 | "max": 40013264, 96 | "avg": 40013264, 97 | "sum": 40013264, 98 | "sum_of_squares": 1601061295933696, 99 | "variance": 0, 100 | "std_deviation": 0 101 | } 102 | }, { 103 | "key_as_string": "2013-12-29T16:00:00.000Z", 104 | "key": 1388332800000, 105 | "doc_count": 0, 106 | "used_memory": { 107 | "count": 0, 108 | "min": null, 109 | "max": null, 110 | "avg": null, 111 | "sum": null, 112 | "sum_of_squares": null, 113 | "variance": null, 114 | "std_deviation": null 115 | } 116 | }, { 117 | "key_as_string": "2013-12-29T16:01:00.000Z", 118 | "key": 1388332860000, 119 | "doc_count": 0, 120 | "used_memory": { 121 | "count": 0, 122 | "min": null, 123 | "max": null, 124 | "avg": null, 125 | "sum": null, 126 | "sum_of_squares": null, 127 | "variance": null, 128 | "std_deviation": null 129 | } 130 | }] 131 | } 132 | } 133 | }, 134 | // ?aggs=avg&aggs=stats&field=used_memory&field=used_memory_rss&interval=minute&start_ts=0... 135 | multiple_fields_used_memory_and_used_memory_rss_agg_avg_and_stats: { 136 | "took": 5, 137 | "timed_out": false, 138 | "_shards": { 139 | "total": 3, 140 | "successful": 3, 141 | "failed": 0 142 | }, 143 | "hits": { 144 | "total": 347, 145 | "max_score": 0, 146 | "hits": [] 147 | }, 148 | "aggregations": { 149 | "volume": { 150 | "buckets": [{ 151 | "key_as_string": "2013-12-29T15:54:00.000Z", 152 | "key": 1388332440000, 153 | "doc_count": 1, 154 | "used_memory": { 155 | "value": 3626992 156 | }, 157 | "used_memory_rss": { 158 | "count": 1, 159 | "min": 2912256, 160 | "max": 2912256, 161 | "avg": 2912256, 162 | "sum": 2912256, 163 | "sum_of_squares": 8481235009536, 164 | "variance": 0, 165 | "std_deviation": 0 166 | } 167 | }, { 168 | "key_as_string": "2013-12-29T15:55:00.000Z", 169 | "key": 1388332500000, 170 | "doc_count": 1, 171 | "used_memory": { 172 | "value": 3177480 173 | }, 174 | "used_memory_rss": { 175 | "count": 1, 176 | "min": 3177480, 177 | "max": 3177480, 178 | "avg": 3177480, 179 | "sum": 3177480, 180 | "sum_of_squares": 10096379150400, 181 | "variance": 0, 182 | "std_deviation": 0 183 | } 184 | }, { 185 | "key_as_string": "2013-12-29T15:56:00.000Z", 186 | "key": 1388332560000, 187 | "doc_count": 1, 188 | "used_memory": { 189 | "value": 1197328496 190 | }, 191 | "used_memory_rss": { 192 | "count": 1, 193 | "min": 1225158656, 194 | "max": 1225158656, 195 | "avg": 1225158656, 196 | "sum": 1225158656, 197 | "sum_of_squares": 1501013732371726300, 198 | "variance": 0, 199 | "std_deviation": 0 200 | } 201 | }, { 202 | "key_as_string": "2013-12-29T15:57:00.000Z", 203 | "key": 1388332620000, 204 | "doc_count": 1, 205 | "used_memory": { 206 | "value": 14033800 207 | }, 208 | "used_memory_rss": { 209 | "count": 1, 210 | "min": 14033800, 211 | "max": 14033800, 212 | "avg": 14033800, 213 | "sum": 14033800, 214 | "sum_of_squares": 196947542440000, 215 | "variance": 0, 216 | "std_deviation": 0 217 | } 218 | }, { 219 | "key_as_string": "2013-12-29T15:58:00.000Z", 220 | "key": 1388332680000, 221 | "doc_count": 0, 222 | "used_memory": { 223 | "value": null 224 | }, 225 | "used_memory_rss": { 226 | "count": 0, 227 | "min": null, 228 | "max": null, 229 | "avg": null, 230 | "sum": null, 231 | "sum_of_squares": null, 232 | "variance": null, 233 | "std_deviation": null 234 | } 235 | }, { 236 | "key_as_string": "2013-12-29T15:59:00.000Z", 237 | "key": 1388332740000, 238 | "doc_count": 1, 239 | "used_memory": { 240 | "value": 40013264 241 | }, 242 | "used_memory_rss": { 243 | "count": 1, 244 | "min": 49119232, 245 | "max": 49119232, 246 | "avg": 49119232, 247 | "sum": 49119232, 248 | "sum_of_squares": 2412698952269824, 249 | "variance": 0, 250 | "std_deviation": 0 251 | } 252 | }, { 253 | "key_as_string": "2013-12-29T16:00:00.000Z", 254 | "key": 1388332800000, 255 | "doc_count": 0, 256 | "used_memory": { 257 | "value": null 258 | }, 259 | "used_memory_rss": { 260 | "count": 0, 261 | "min": null, 262 | "max": null, 263 | "avg": null, 264 | "sum": null, 265 | "sum_of_squares": null, 266 | "variance": null, 267 | "std_deviation": null 268 | } 269 | }, { 270 | "key_as_string": "2013-12-29T16:01:00.000Z", 271 | "key": 1388332860000, 272 | "doc_count": 0, 273 | "used_memory": { 274 | "value": null 275 | }, 276 | "used_memory_rss": { 277 | "count": 0, 278 | "min": null, 279 | "max": null, 280 | "avg": null, 281 | "sum": null, 282 | "sum_of_squares": null, 283 | "variance": null, 284 | "std_deviation": null 285 | } 286 | }, { 287 | "key_as_string": "2013-12-29T16:02:00.000Z", 288 | "key": 1388332920000, 289 | "doc_count": 1, 290 | "used_memory": { 291 | "value": 3230648 292 | }, 293 | "used_memory_rss": { 294 | "count": 1, 295 | "min": 2609152, 296 | "max": 2609152, 297 | "avg": 2609152, 298 | "sum": 2609152, 299 | "sum_of_squares": 6807674159104, 300 | "variance": 0, 301 | "std_deviation": 0 302 | } 303 | }, { 304 | "key_as_string": "2013-12-29T16:03:00.000Z", 305 | "key": 1388332980000, 306 | "doc_count": 1, 307 | "used_memory": { 308 | "value": 3527064 309 | }, 310 | "used_memory_rss": { 311 | "count": 1, 312 | "min": 3527064, 313 | "max": 3527064, 314 | "avg": 3527064, 315 | "sum": 3527064, 316 | "sum_of_squares": 12440180460096, 317 | "variance": 0, 318 | "std_deviation": 0 319 | } 320 | }, { 321 | "key_as_string": "2013-12-29T16:04:00.000Z", 322 | "key": 1388333040000, 323 | "doc_count": 0, 324 | "used_memory": { 325 | "value": null 326 | }, 327 | "used_memory_rss": { 328 | "count": 0, 329 | "min": null, 330 | "max": null, 331 | "avg": null, 332 | "sum": null, 333 | "sum_of_squares": null, 334 | "variance": null, 335 | "std_deviation": null 336 | } 337 | }, { 338 | "key_as_string": "2013-12-29T16:05:00.000Z", 339 | "key": 1388333100000, 340 | "doc_count": 1, 341 | "used_memory": { 342 | "value": 25079536 343 | }, 344 | "used_memory_rss": { 345 | "count": 1, 346 | "min": 27361280, 347 | "max": 27361280, 348 | "avg": 27361280, 349 | "sum": 27361280, 350 | "sum_of_squares": 748639643238400, 351 | "variance": 0, 352 | "std_deviation": 0 353 | } 354 | }, { 355 | "key_as_string": "2013-12-29T16:06:00.000Z", 356 | "key": 1388333160000, 357 | "doc_count": 1, 358 | "used_memory": { 359 | "value": 3172621 360 | }, 361 | "used_memory_rss": { 362 | "count": 1, 363 | "min": 3172621, 364 | "max": 3172621, 365 | "avg": 3172621, 366 | "sum": 3172621, 367 | "sum_of_squares": 10065524009641, 368 | "variance": 0, 369 | "std_deviation": 0 370 | } 371 | }, { 372 | "key_as_string": "2013-12-29T16:07:00.000Z", 373 | "key": 1388333220000, 374 | "doc_count": 0, 375 | "used_memory": { 376 | "value": null 377 | }, 378 | "used_memory_rss": { 379 | "count": 0, 380 | "min": null, 381 | "max": null, 382 | "avg": null, 383 | "sum": null, 384 | "sum_of_squares": null, 385 | "variance": null, 386 | "std_deviation": null 387 | } 388 | }] 389 | } 390 | } 391 | } 392 | }; 393 | -------------------------------------------------------------------------------- /api/domain/MeasurementQuery.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | require('../../bootstrap.test'); 3 | 4 | var DateRangeInterval = require('./DateRangeInterval.ValueObject')(); 5 | var MeasurementQuery = require('./MeasurementQuery.ValueObject')(DateRangeInterval); 6 | var Fixtures = require('./MeasurementQuery.test.fixture'); 7 | 8 | 9 | describe('MeasurementQuery', function () { 10 | var dateRangeInterval; 11 | beforeEach(function () { 12 | dateRangeInterval = new DateRangeInterval(1420293008365, 1420294008365, 'minute'); 13 | }); 14 | 15 | describe('MeasurementQuery.fromReq', function () { 16 | it('should return an error if the query is not defined', function () { 17 | t.isPrettyError(MeasurementQuery.fromReq(), 400); 18 | }); 19 | 20 | it('should return an error if the interval is not defined', function () { 21 | t.ok(MeasurementQuery.fromReq({ 22 | query: {} 23 | }) instanceof PrettyError, 'should be a pretty error'); 24 | // @todo check error.details 25 | }); 26 | 27 | it('should return an error if id or ids are not present', function () { 28 | var err = MeasurementQuery.fromReq({ 29 | query: { 30 | interval: 'plop' 31 | } 32 | }, dateRangeInterval); 33 | t.ok(err instanceof PrettyError, 'should be a pretty error'); 34 | t.include(err.message, 'id or ids'); 35 | }); 36 | 37 | it('should return an error if field or fields are not present', function () { 38 | var err = MeasurementQuery.fromReq({ 39 | query: { 40 | interval: 'plop', 41 | id: '10' 42 | } 43 | }, dateRangeInterval); 44 | t.ok(err instanceof PrettyError, 'should be a pretty error'); 45 | t.include(err.message, 'field or fields'); 46 | }); 47 | 48 | it('should return an error if filter or filters exists and is not a string', function () { 49 | var err = MeasurementQuery.fromReq({ 50 | query: { 51 | interval: 'plop', 52 | id: '10', 53 | field: 'a', 54 | agg: 'a', 55 | filter: 3 56 | } 57 | }, dateRangeInterval); 58 | t.ok(err instanceof PrettyError, 'should be a pretty error'); 59 | t.include(err.message, 'filter or filters'); 60 | }); 61 | 62 | it('should return return an error if the specified aggregates don\'t match the fields count', function () { 63 | var err = MeasurementQuery.fromReq({ 64 | query: { 65 | interval: 'plop', 66 | id: '10', 67 | fields: ['a', 'b'], 68 | agg: 'a' 69 | } 70 | }, dateRangeInterval); 71 | t.ok(err instanceof PrettyError, 'should be a pretty error'); 72 | t.include(err.message, 'specified an aggregation type'); 73 | }); 74 | 75 | it('should return specify a default aggregate the same size as fields', function () { 76 | var measurementQuery = MeasurementQuery.fromReq({ 77 | query: { 78 | id: '10', 79 | fields: ['plop', 'plop'] 80 | } 81 | }, dateRangeInterval); 82 | t.deepEqual(measurementQuery.aggs, ['avg', 'avg']); 83 | }); 84 | }); 85 | 86 | describe('MeasurementQuery', function () { 87 | var measurementQuery; 88 | 89 | 90 | describe('::buildQuery', function () { 91 | beforeEach(function () { 92 | measurementQuery = MeasurementQuery.fromReq({ 93 | query: { 94 | id: '10', 95 | fields: ['plop', 'plop'], 96 | filters: 'data.server_id == "549db2d721a4764672000397" and data.used_memory > 9000' 97 | } 98 | }, dateRangeInterval); 99 | }); 100 | 101 | it('should return an query', function () { 102 | var makeIndexFromId = _.identity; 103 | var index_document_type = 'plop'; 104 | var query = measurementQuery.buildQuery(makeIndexFromId, index_document_type); 105 | t.strictEqual(query.type, index_document_type); 106 | t.deepEqual(query, { 107 | "indices": ["10"], 108 | "type": "plop", 109 | "fields": ["plop", "plop"], 110 | "search_type": "count", 111 | "body": { 112 | "size": 0, 113 | "query": { 114 | "filtered": { 115 | "filter": { 116 | "bool": { 117 | "must": [{ 118 | "range": { 119 | "timestamp": { 120 | "from": 1420293008365, 121 | "to": 1420294008365 122 | } 123 | } 124 | }, 125 | [{ 126 | "bool": { 127 | "must": [{ 128 | "term": { 129 | "data.server_id": "549db2d721a4764672000397" 130 | } 131 | }, { 132 | "range": { 133 | "data.used_memory": { 134 | "gt": 9000 135 | } 136 | } 137 | }] 138 | } 139 | }] 140 | ] 141 | } 142 | } 143 | } 144 | }, 145 | "aggs": { 146 | "volume": { 147 | "date_histogram": { 148 | "field": "timestamp", 149 | "min_doc_count": 0, 150 | "interval": "minute", 151 | "extended_bounds": { 152 | "min": 1420293008365, 153 | "max": 1420294008365 154 | } 155 | }, 156 | "aggs": { 157 | "plop": { 158 | "avg": { 159 | "field": "plop" 160 | } 161 | } 162 | } 163 | } 164 | } 165 | } 166 | }); 167 | }); 168 | }); 169 | 170 | describe('::parseResults', function () { 171 | 172 | describe('... with a single field', function () { 173 | var result; 174 | beforeEach(function () { 175 | measurementQuery = MeasurementQuery.fromReq({ 176 | query: { 177 | id: '10', 178 | field: 'used_memory', 179 | agg: 'avg' 180 | } 181 | }, dateRangeInterval); 182 | result = measurementQuery.parseResults(Fixtures.single_field_used_memory_agg_stats); 183 | }); 184 | 185 | it('should return an array', function () { 186 | t.isArray(result); 187 | }); 188 | 189 | it('should return an array or object', function () { 190 | result.every(_.isObject); 191 | }); 192 | 193 | it('should return a single object (1 field and 1 series)', function () { 194 | assert(measurementQuery.ids.length, 1); 195 | t.strictEqual(result.length, measurementQuery.ids.length); 196 | }); 197 | 198 | it('should return an object for each series', function () { 199 | var firstResult = _.chain(result).first().value(); 200 | 201 | t.deepEqual(_.omit(firstResult, 'values'), { 202 | id: measurementQuery.ids[0], 203 | field: measurementQuery.fields[0] 204 | }); 205 | }); 206 | 207 | // @todo test values 208 | }); 209 | 210 | describe('... with multiple fields', function () { 211 | var result; 212 | beforeEach(function () { 213 | measurementQuery = MeasurementQuery.fromReq({ 214 | query: { 215 | id: '10', 216 | fields: ['used_memory', 'used_memory_rss'], 217 | aggs: ['avg', 'stats'] 218 | } 219 | }, dateRangeInterval); 220 | result = measurementQuery.parseResults(Fixtures.multiple_fields_used_memory_and_used_memory_rss_agg_avg_and_stats); 221 | }); 222 | 223 | it('should return 2 root objects (1 field and 1 series)', function () { 224 | assert(measurementQuery.ids.length, 1); 225 | t.strictEqual(result.length, measurementQuery.fields.length); 226 | }); 227 | 228 | describe('the first main object field=used_memory, agg=avg', function () { 229 | var obj; 230 | beforeEach(function () { 231 | obj = _.chain(result).head().value(); 232 | }); 233 | 234 | it('should be an object with the right id and field', function () { 235 | t.deepEqual(_.omit(obj, 'values'), { 236 | id: '10', 237 | field: 'used_memory' 238 | }); 239 | }); 240 | 241 | it('should have a values array', function () { 242 | t.isArray(obj.values); 243 | }); 244 | 245 | it('should have a values array and each values should have `avg` aggregate attributes', function () { 246 | t.deepEqual(_.first(obj.values), { 247 | timestamp: 1388332440000, 248 | value: 3626992 249 | }); 250 | }); 251 | }); 252 | 253 | describe('the first main object field=used_memory_rss, agg=stats', function () { 254 | var obj; 255 | beforeEach(function () { 256 | obj = _.chain(result).rest().head().value(); 257 | }); 258 | 259 | it('should be an object with the right id and field', function () { 260 | t.deepEqual(_.omit(obj, 'values'), { 261 | id: '10', 262 | field: 'used_memory_rss' 263 | }); 264 | }); 265 | 266 | it('should have a values array', function () { 267 | t.isArray(obj.values); 268 | }); 269 | 270 | it('should have a values array and each values should have `avg` aggregate attributes', function () { 271 | t.deepEqual(_.first(obj.values), { 272 | "timestamp": 1388332440000, 273 | "count": 1, 274 | "min": 2912256, 275 | "max": 2912256, 276 | "avg": 2912256, 277 | "sum": 2912256, 278 | "sum_of_squares": 8481235009536, 279 | "variance": 0, 280 | "std_deviation": 0 281 | }); 282 | }); 283 | }); 284 | }); 285 | }); 286 | }); 287 | }); 288 | -------------------------------------------------------------------------------- /api/domain/Template.ValueObject.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /** 3 | * ElasticSearch Template value 4 | */ 5 | 6 | module.exports = function (es, config, template) { 7 | assert(_.isString(config.elasticsearch.index.name_prefix)); 8 | assert(_.isString(config.elasticsearch.index.document.type)); 9 | 10 | assert(_.isNumber(config.elasticsearch.index.settings.number_of_shards)); 11 | assert(_.isNumber(config.elasticsearch.index.settings.number_of_replicas)); 12 | 13 | /** 14 | * [Template description] 15 | * @param {Date} start_date 16 | * @param {Date} end_date 17 | * @param {String} interval 18 | */ 19 | function Template() { 20 | 21 | } 22 | 23 | Template.setupTemplate = function (f) { 24 | var TEMPLATE = _.cloneDeep(template.base); 25 | var USER_DEFINED_TEMPLATE = _.cloneDeep(template.defined); 26 | 27 | // Setup template 28 | var INDEX_NAME_PREFIX = config.elasticsearch.index.name_prefix; 29 | var INDEX_DOCUMENT_TYPE = config.elasticsearch.index.document.type; 30 | 31 | // Change template name 32 | TEMPLATE.template = TEMPLATE.template.replace('{{INDEX_NAME_PREFIX}}', INDEX_NAME_PREFIX); 33 | TEMPLATE.name = TEMPLATE.name.replace('{{INDEX_NAME_PREFIX}}', INDEX_NAME_PREFIX); 34 | 35 | // Change settings 36 | TEMPLATE.settings.number_of_replicas = config.elasticsearch.index.settings.number_of_replicas; 37 | TEMPLATE.settings.number_of_shards = config.elasticsearch.index.settings.number_of_shards; 38 | 39 | // Change document type 40 | var mapping = TEMPLATE.mappings['{{INDEX_DOCUMENT_TYPE}}']; 41 | var newMapping = {}; 42 | newMapping[INDEX_DOCUMENT_TYPE] = mapping; 43 | TEMPLATE.mappings = newMapping; 44 | 45 | // Add user defined mapping 46 | mapping.dynamic_templates = mapping.dynamic_templates.concat(USER_DEFINED_TEMPLATE.dynamic_templates || []); 47 | mapping.properties.data.properties = USER_DEFINED_TEMPLATE.data_mapping || {}; 48 | mapping.properties.metadata.properties = USER_DEFINED_TEMPLATE.metadata_mapping || {}; 49 | 50 | // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-templates.html#delete 51 | es.indices.deleteTemplate({ 52 | name: TEMPLATE.name 53 | }, 54 | function (err, body, code) { 55 | 56 | if (err && code !== 404) { 57 | return f(err); 58 | } 59 | 60 | // Create an index template that will automatically be applied to new indices created. 61 | // https://github.com/elasticsearch/elasticsearch-js/blob/3df3b09d3abf140fe8446b4e19d0014cc8545117/src/lib/apis/1_3.js#L3251 62 | es.indices.putTemplate({ 63 | name: TEMPLATE.name, 64 | body: TEMPLATE 65 | }, function (err, body) { 66 | f(err, body); 67 | }); 68 | }); 69 | }; 70 | 71 | return Template; 72 | }; 73 | -------------------------------------------------------------------------------- /api/domain/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function (es, amqp, config, template) { 4 | var DateRangeInterval = require('./DateRangeInterval.ValueObject')(); 5 | var MeasurementQuery = require('./MeasurementQuery.ValueObject')(DateRangeInterval); 6 | 7 | return { 8 | // Entities 9 | // entities can only require other entities 10 | Measurement: require('./Measurement.Entity'), 11 | 12 | // Repositories 13 | // repository can require es, amqp and other entities 14 | Measurements: require('./Measurement.Repository')(es, amqp, config, DateRangeInterval, MeasurementQuery), 15 | 16 | // Value Object 17 | DateRangeInterval: DateRangeInterval, 18 | Template: require('./Template.ValueObject')(es, config, template), 19 | MeasurementQuery: MeasurementQuery 20 | }; 21 | }; 22 | -------------------------------------------------------------------------------- /api/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = require('./api'); 4 | -------------------------------------------------------------------------------- /api/middlewares/augmentReqAndRes.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * [exports description] 5 | * @param {Function} onError(err) 6 | */ 7 | module.exports = function (onError) { 8 | return function (req, res, next) { 9 | 10 | req.p = function (param) { 11 | return req.params[param] || req.param(param); 12 | }; 13 | 14 | res.errorOrValue = function (err, value) { 15 | if (err) { 16 | return res.error(err); 17 | } 18 | 19 | return res.ok(value); 20 | }; 21 | 22 | // For express 3.x res.json(code, json); 23 | /** 24 | * [error description] 25 | * 26 | * Usage: 27 | * res.error(new PrettyError(500, '', err)); 28 | * res.error(401, new PrettyError(500, '', err)); 29 | * 30 | * @param {Number} code 31 | * @param {PrettyError} error 32 | */ 33 | res.error = function (code, error) { 34 | if (code instanceof PrettyError) { 35 | error = code; 36 | code = error.code || 500; 37 | } 38 | 39 | onError(error, req.method, req.url, req); 40 | return res.status(code).json(PrettyError.ErrorToJSON(error) || {}); 41 | }; 42 | 43 | res.ok = function (jsonOrError, code) { 44 | return res.status(code || 200).json(jsonOrError); 45 | }; 46 | 47 | next(); 48 | }; 49 | }; 50 | -------------------------------------------------------------------------------- /api/routes.js: -------------------------------------------------------------------------------- 1 | /*jshint -W098 */ 2 | 'use strict'; 3 | 4 | module.exports = function (logger, es, amqp, fOnError, domain) { 5 | var augmentReqAndRes = require('./middlewares/augmentReqAndRes'); 6 | var controllers = require('./controllers')(domain); 7 | 8 | return function (app) { 9 | app.use(augmentReqAndRes(fOnError)); 10 | 11 | app.get('/api/v1/measurements', controllers.measurements.get); 12 | 13 | // ensure that measurement id is well defined 14 | app.all('/api/v1/measurements/:measurement_id', controllers.measurements.middlewares); 15 | 16 | app.post('/api/v1/measurements/:measurement_id', controllers.measurements.post); 17 | app.get('/api/v1/measurements/:measurement_id/describe', controllers.measurements.describe); 18 | 19 | app.delete('/internal/removeAllData', controllers.measurements.removeAllData); 20 | app.all('/internal/templates/setup', controllers.templates.setup); 21 | 22 | app.use(function errorHandler(err, req, res, next) { 23 | var publicError = new PrettyError(500, 'Internal Server Error', err); 24 | // clean stack because it does not say much 25 | publicError.stack = null; 26 | res.status(500).send(publicError); 27 | }); 28 | }; 29 | }; 30 | -------------------------------------------------------------------------------- /benchmark/run.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | require('../bootstrap'); 3 | var RAW_DATA = require('./data.json'); 4 | 5 | var logger = require('../helpers/logger'); 6 | var config = require('../config')(logger); 7 | var request = require('requestretry'); 8 | var flatten = require('flat'); 9 | 10 | var data = RAW_DATA.map(function (obj) { 11 | // remove data we don't need 12 | // we will generate an id and a ts automatically 13 | var cleanObject = _.omit(obj, '_id', '_serverId', 'ts'); 14 | return flatten(cleanObject); 15 | }); 16 | 17 | var ONE_YEAR = 365 * 24 * 3600 * 1000; 18 | 19 | var CONCURRENCY = 20; 20 | var NUM_REQUEST = 1000; 21 | var NUM_UUID = 4; 22 | var UUIDS = _.range(1, NUM_UUID).map(generateUUID); 23 | 24 | // prepare 25 | console.log('precomputing requests'); 26 | var ids = _.range(1, NUM_REQUEST).map(function (i) { 27 | return [i, { 28 | id: _.sample(UUIDS), 29 | timestamp: +new Date() - ONE_YEAR + i * 60 * 1000, 30 | data: _.sample(data) 31 | }]; 32 | }); 33 | console.log('ready'); 34 | 35 | console.time('Benchmark total'); 36 | async.eachLimit(ids, CONCURRENCY, sendData, function () { 37 | console.timeEnd('Benchmark total'); 38 | }); 39 | 40 | 41 | function sendData(pair, f) { 42 | var i = pair[0]; 43 | var body = pair[1]; 44 | var TIME = i + ' ' + body.id + '-' + body.timestamp; 45 | console.time(TIME); 46 | // console.log(JSON.stringify(body, null, 2)); 47 | // return; 48 | 49 | request({ 50 | url: 'http://localhost:' + config.api.port + '/api/v1/measurements', 51 | method: 'POST', 52 | json: true, 53 | body: body, 54 | maxAttempts: 1, 55 | retryDelay: 5000, 56 | retryStrategy: request.RetryStrategies.NetworkError 57 | }, function (err, resp, body) { 58 | console.timeEnd(TIME); 59 | f(err); 60 | }); 61 | } 62 | 63 | function generateUUID() { 64 | var d = new Date().getTime(); 65 | var uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { 66 | var r = (d + Math.random() * 16) % 16 | 0; 67 | d = Math.floor(d / 16); 68 | return (c == 'x' ? r : (r & 0x3 | 0x8)).toString(16); 69 | }); 70 | return uuid; 71 | } 72 | -------------------------------------------------------------------------------- /bootstrap.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // We use these library all the time, move them from user-land to standard lib. 4 | 5 | global.async = require('async'); 6 | global.assert = require('better-assert'); 7 | global._ = require('lodash'); 8 | global.PrettyError = require('./helpers/PrettyError'); 9 | _.mixin(require('./helpers/validation')); 10 | -------------------------------------------------------------------------------- /bootstrap.test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /* 4 | Require this file inside tests 5 | */ 6 | require('./bootstrap'); 7 | 8 | var fs = require('fs'); 9 | 10 | if (!process.env.OVERRIDE) { 11 | _.compact(fs.readFileSync('./ci/.env_test').toString('utf8').split('\n')).forEach(function (line) { 12 | var pair = _.rest(line.split('export ')).join(' ').split('=').map(function trim(v) { 13 | return v.trim(); 14 | }); 15 | 16 | process.env[pair[0]] = pair[1].replace(/"/g, ''); 17 | }); 18 | } 19 | 20 | // ensure that we are running the tests in dev env 21 | 22 | var ServerFactory = require('./server'); 23 | global.t = require('chai').assert; 24 | global.request = require('supertest'); 25 | 26 | // Extend 27 | t.isPrettyError = function (err, code, message) { 28 | t.instanceOf(err, PrettyError); 29 | t.ok(err instanceof PrettyError, err.toJSON()); 30 | t.strictEqual(err.code, code, 'pretty error ' + err.toString() + ' code should be code=' + code); 31 | 32 | if (message) { 33 | t.include(err.message, message); 34 | } 35 | }; 36 | 37 | /** 38 | * @param {Function} f(app, config, logger, es, amqp) 39 | */ 40 | t.getAPP = function getAPP(f) { 41 | ServerFactory(function (app, config, logger, es, amqp) { 42 | // ensure that we are using test configuration 43 | 44 | f(app, config, logger, es, amqp); 45 | }, _.noop, { 46 | amqpError: true 47 | }); 48 | }; 49 | -------------------------------------------------------------------------------- /ci/.env_test: -------------------------------------------------------------------------------- 1 | export ELASTICSEARCH_HOST_HOST="redsmintest.west-eu.azr.facetflow.io" 2 | export ELASTICSEARCH_HOST_PROTOCOL="https" 3 | export ELASTICSEARCH_HOST_AUTH="b8rgsz43skO5F7WzZrvUdAW64RUYpPqk:" 4 | export ELASTICSEARCH_HOST_PORT=443 5 | 6 | export AMQP_HOST="rabbitmq.redsmin.com" 7 | export AMQP_LOGIN="redsmin_test" 8 | export AMQP_VHOST="test" 9 | export AMQP_PASSWORD="gtjfxrv4QNQVLw4cjBPdU5rnBE9rm7_3" 10 | 11 | export ELASTICSEARCH_INDEX_SETTINGS_NUMBER_OF_SHARDS=1 12 | export ELASTICSEARCH_INDEX_SETTINGS_NUMBER_OF_REPLICAS=0 13 | -------------------------------------------------------------------------------- /ci/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/statwarn/monitoring-api/0a571b21518f89d377329835d1f2660710cc5fb4/ci/.gitkeep -------------------------------------------------------------------------------- /ci/ci-start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | export NODE_ENV=preprod 3 | DIR=${0%/*} 4 | cd $DIR/.. 5 | wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.21.0/install.sh | bash 6 | source ~/.nvm/nvm.sh 7 | nvm install 8 | nvm use 9 | npm install 10 | npm install 11 | set -e 12 | # Any subsequent commands which fail will cause the shell script to exit immediately 13 | setopt extended_glob; 14 | # Allow extended globbing (see fellow) 15 | `npm bin`/mocha -t 10000 -R spec **/*.test.js~node_modules/*; 16 | npm install -g check-build 17 | check-build 18 | -------------------------------------------------------------------------------- /config/config.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = function (logger) { 4 | var env = require('common-env')(logger); 5 | 6 | var config = env.getOrElseAll({ 7 | statwarn: { 8 | schema: { 9 | monitoring: { 10 | create: 'application/vnd.com.statwarn.monitoring.create.v1+json' 11 | } 12 | }, 13 | monitoring: { 14 | api: { 15 | port: 9002 16 | } 17 | } 18 | }, 19 | 20 | elasticsearch: { 21 | host: { 22 | protocol: 'http', 23 | host: 'localhost', 24 | port: 9200, 25 | auth: '' 26 | }, 27 | 28 | // String, String[], Object, Object[], Constructor — Unless a constructor is specified, this sets the output settings for the bundled logger. See the section on configuring-logging[logging] for more information. 29 | // error, warning, info, debug, trace 30 | // Event fired for "info" level log entries, which usually describe what a client is doing (sniffing etc) 31 | // Event fired for "debug" level log entries, which will describe requests sent, 32 | log: 'info', 33 | 34 | // Integer — How many times should the client try to connect to other nodes before returning a ConnectionFault error. 35 | maxRetries: 5, 36 | 37 | // Number — Milliseconds before an HTTP request will be aborted and retried. This can also be set per request. 38 | requestTimeout: 10000, 39 | 40 | // Number — Milliseconds that a dead connection will wait before attempting to revive itself. 41 | deadTimeout: 1000, 42 | index: { 43 | template: 'defined/redsmin.template.js', 44 | settings: { 45 | number_of_shards: 1, 46 | number_of_replicas: 0 47 | }, 48 | name_prefix: 'statwarn', 49 | document: { 50 | type: 'measurement' 51 | } 52 | } 53 | }, 54 | 55 | amqp: { 56 | login: 'guest', 57 | password: 'guest', 58 | host: 'localhost', 59 | port: 5672, 60 | vhost: '', 61 | 62 | publish: { 63 | // Publish new measurement on exchange 64 | exchange: 'monitoring', 65 | publish_key: 'monitoring.create' 66 | } 67 | } 68 | }); 69 | 70 | // export env 71 | config.env = env; 72 | 73 | return config; 74 | }; 75 | -------------------------------------------------------------------------------- /config/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | module.exports = require('./config'); 3 | -------------------------------------------------------------------------------- /docs/Elasticsearch.md: -------------------------------------------------------------------------------- 1 | # Schema 2 | 3 | - Index 4 | + Document 5 | 6 | - Index name : "monitoring-:server_id" 7 | - Document : stocker le JSON retourné par info. 8 | 9 | -> analyzer/schéma 10 | 11 | # Document in : 12 | 13 | { 14 | "redsmin_version":"129.12" 15 | } 16 | 17 | # Analyzer 18 | 19 | "content": { 20 | "type": "string", 21 | "index": "analyzed", 22 | "analyzer": "social_analyzer", 23 | "fields": { 24 | "urls": { 25 | "type": "string", 26 | "index": "analyzed", 27 | "analyzer": "url_analyzer" 28 | } 29 | } 30 | }, 31 | 32 | - Index name : "monitoring-:server_id" 33 | - document : stocker le JSON retourné par redis info. 34 | 35 | PUT /_template/tmpl_monitoring 36 | 37 | { 38 | "template": "monitoring-*", 39 | "settings": { 40 | "number_of_shards": 1, 41 | "number_of_replicas": 1 42 | }, 43 | "mappings": { 44 | "info": { 45 | "_timestamp": { 46 | "enabled": "true", 47 | "path": "created_at" 48 | }, 49 | "_all": { 50 | "enabled": "false" 51 | }, 52 | "_source": { 53 | "enabled": "true" 54 | }, 55 | "dynamic": "strict", 56 | "properties": { 57 | "server_id": { 58 | "type": "string", 59 | "index": "not_analyzed" 60 | }, 61 | "created_at": { 62 | "type": "date", 63 | "index": "not_analyzed" 64 | }, 65 | "metrics": { 66 | "type": "object", 67 | "properties": { 68 | "uptime_in_seconds": { 69 | "type": "date", 70 | "index": "not_analyzed" 71 | }, 72 | "connected_clients": { 73 | "type": "integer", 74 | "index": "not_analyzed" 75 | }, 76 | "client_longest_output_list": { 77 | "type": "integer", 78 | "index": "not_analyzed" 79 | }, 80 | "client_biggest_input_buf": { 81 | "type": "integer", 82 | "index": "not_analyzed" 83 | }, 84 | "blocked_clients": { 85 | "type": "integer", 86 | "index": "not_analyzed" 87 | }, 88 | "used_memory": { 89 | "type": "integer", 90 | "index": "not_analyzed" 91 | }, 92 | "used_memory_rss": { 93 | "type": "integer", 94 | "index": "not_analyzed" 95 | }, 96 | "used_memory_peak": { 97 | "type": "integer", 98 | "index": "not_analyzed" 99 | }, 100 | "used_memory_lua": { 101 | "type": "integer", 102 | "index": "not_analyzed" 103 | }, 104 | "mem_fragmentation_ratio": { 105 | "type": "float", 106 | "index":"not_analyzed" 107 | }, 108 | "rdb_changes_since_last_save": { 109 | "type": "date", 110 | "index": "not_analyzed" 111 | }, 112 | "rdb_last_save_time": { 113 | "type": "integer", 114 | "index": "not_analyzed" 115 | }, 116 | "rdb_last_bgsave_time_sec": { 117 | "type": "integer", 118 | "index": "not_analyzed" 119 | }, 120 | "rdb_current_bgsave_time_sec": { 121 | "type": "integer", 122 | "index": "not_analyzed" 123 | }, 124 | "aof_rewrite_in_progress": { 125 | "type": "integer", 126 | "index": "not_analyzed" 127 | }, 128 | "aof_rewrite_scheduled": { 129 | "type": "integer", 130 | "index": "not_analyzed" 131 | }, 132 | "aof_last_rewrite_time_sec": { 133 | "type": "integer", 134 | "index": "not_analyzed" 135 | }, 136 | "aof_current_rewrite_time_sec": { 137 | "type": "integer", 138 | "index": "not_analyzed" 139 | }, 140 | "total_connections_received": { 141 | "type": "integer", 142 | "index": "not_analyzed" 143 | }, 144 | "total_commands_processed": { 145 | "type": "integer", 146 | "index": "not_analyzed" 147 | }, 148 | "instantaneous_ops_per_sec": { 149 | "type": "integer", 150 | "index": "not_analyzed" 151 | }, 152 | "rejected_connections": { 153 | "type": "integer", 154 | "index": "not_analyzed" 155 | }, 156 | "expired_keys": { 157 | "type": "integer", 158 | "index": "not_analyzed" 159 | }, 160 | "evicted_keys": { 161 | "type": "integer", 162 | "index": "not_analyzed" 163 | }, 164 | "keyspace_hits": { 165 | "type": "integer", 166 | "index": "not_analyzed" 167 | }, 168 | "keyspace_misses": { 169 | "type": "integer", 170 | "index": "not_analyzed" 171 | }, 172 | "pubsub_channels": { 173 | "type": "integer", 174 | "index": "not_analyzed" 175 | }, 176 | "pubsub_patterns": { 177 | "type": "integer", 178 | "index": "not_analyzed" 179 | }, 180 | "latest_fork_usec": { 181 | "type": "integer", 182 | "index": "not_analyzed" 183 | }, 184 | "connected_slaves": { 185 | "type": "integer", 186 | "index": "not_analyzed" 187 | }, 188 | "used_cpu_sys": { 189 | "type": "float", 190 | "index": "not_analyzed" 191 | }, 192 | "used_cpu_user": { 193 | "type": "float", 194 | "index": "not_analyzed" 195 | }, 196 | "commands": { 197 | "type": "object", 198 | "properties": { 199 | "*": { 200 | "properties": { 201 | "calls": { 202 | "type:": "integer", 203 | "index": "not_analyzed" 204 | }, 205 | "usec": { 206 | "type:": "integer", 207 | "index": "not_analyzed" 208 | }, 209 | "usec_per_call": { 210 | "type:": "float", 211 | "index": "not_analyzed" 212 | } 213 | } 214 | } 215 | } 216 | }, 217 | "databases": { 218 | "type": "object", 219 | "properties": { 220 | "*": { 221 | "properties": { 222 | "keys": { 223 | "type": "integer", 224 | "index": "not_analyzed" 225 | }, 226 | "expires": { 227 | "type": "integer", 228 | "index": "not_analyzed" 229 | } 230 | } 231 | } 232 | } 233 | } 234 | } 235 | } 236 | } 237 | } 238 | } 239 | } 240 | 241 | 5 shards = 3 serveurs 242 | 243 | -------------------------------------------------------------------------------- /docs/es_post_example.js: -------------------------------------------------------------------------------- 1 | { 2 | "info": { 3 | "metrics": { 4 | "uptime_in_seconds": 1525900, 5 | "connected_clients": 1, 6 | "client_longest_output_list": 0, 7 | "client_biggest_input_buf": 0, 8 | "blocked_clients": 0, 9 | "used_memory": 1079416, 10 | "used_memory_rss": 1045760, 11 | "used_memory_peak": 1096608, 12 | "used_memory_lua": 33792, 13 | "mem_fragmentation_ratio": 0.97, 14 | "rdb_changes_since_last_save": 0, 15 | "rdb_last_save_time": 1415957886, 16 | "rdb_last_bgsave_time_sec": -1, 17 | "rdb_current_bgsave_time_sec": -1, 18 | "aof_rewrite_in_progress": 0, 19 | "aof_rewrite_scheduled": 0, 20 | "aof_last_rewrite_time_sec": -1, 21 | "aof_current_rewrite_time_sec": -1, 22 | "total_connections_received": 74, 23 | "total_commands_processed": 10135, 24 | "instantaneous_ops_per_sec": 0, 25 | "rejected_connections": 0, 26 | "expired_keys": 0, 27 | "evicted_keys": 0, 28 | "keyspace_hits": 0, 29 | "keyspace_misses": 0, 30 | "pubsub_channels": 0, 31 | "pubsub_patterns": 0, 32 | "latest_fork_usec": 0, 33 | "connected_slaves": 0, 34 | "used_cpu_sys": 480.31, 35 | "used_cpu_user": 234.64, 36 | "commands": { 37 | "info": { 38 | "calls": 10136, 39 | "usec": 583086, 40 | "usec_per_call": 57.53 41 | } 42 | }, 43 | "databases": { 44 | "0": { 45 | "keys": 50295571, 46 | "expires": 0 47 | }, 48 | "1": { 49 | "keys": 50295572, 50 | "expires": 22 51 | } 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /helpers/PrettyError.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var IS_PROD = process.env.NODE_ENV === "production"; 3 | 4 | function PrettyError(code, message, cause, details) { 5 | this.message = message; 6 | this.code = code; 7 | this.stack = _cleanStack(new Error().stack); 8 | this.cause = cause || null; 9 | this.details = details || null; 10 | } 11 | 12 | function _cleanStack(stack) { 13 | 14 | return stack 15 | .split('\n') 16 | .map(function (line) { 17 | return line.trim(); 18 | }) 19 | .filter(function (line) { 20 | return line.indexOf('new PrettyError') === -1; 21 | }); 22 | } 23 | 24 | function ErrorToJSON(err) { 25 | if (err instanceof Error) { 26 | return { 27 | message: err.message, 28 | stack: _cleanStack(err.stack) 29 | }; 30 | } 31 | 32 | if (err instanceof PrettyError) { 33 | return err.toJSON(); 34 | } 35 | 36 | return err ? err.toString() : null; 37 | } 38 | 39 | PrettyError.ErrorToJSON = ErrorToJSON; 40 | 41 | PrettyError.fromValidation = function (validationError) { 42 | var errors = validationError.errors.map(function (error) { 43 | // remove stack for production envs. 44 | var err = new PrettyError(error.code, error.message, IS_PROD ? null : error); 45 | err.dataPath = error.dataPath; 46 | return err; 47 | }); 48 | 49 | return new PrettyError(400, 'Validation error', null, errors); 50 | }; 51 | 52 | // todo : create middleware to check if IS_PROD and show stack (or not) 53 | PrettyError.prototype.toJSON = function () { 54 | var o = { 55 | code: this.code, 56 | message: this.message 57 | }; 58 | 59 | if (!IS_PROD) { 60 | o.stack = this.stack; 61 | o.cause = ErrorToJSON(this.cause); 62 | } 63 | 64 | if (this.details) { 65 | o.details = this.details.map(ErrorToJSON); 66 | } 67 | 68 | if (this.dataPath) { 69 | o.dataPath = this.dataPath; 70 | } 71 | 72 | return o; 73 | }; 74 | 75 | PrettyError.prototype.toError = function () { 76 | var err = new Error(this.message); 77 | err.code = this.code; 78 | err.stack = this.stack; 79 | return err; 80 | }; 81 | 82 | PrettyError.prototype.toString = function () { 83 | return JSON.stringify(this.toJSON(), null, 2); 84 | }; 85 | 86 | module.exports = PrettyError; 87 | -------------------------------------------------------------------------------- /helpers/amqp.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var amqp = require('amqp-dsl'); 3 | 4 | module.exports = function (config, logger, options) { 5 | options = _.defaults(options, { 6 | amqpError: false 7 | }); 8 | 9 | // (obj, f) 10 | // (f) 11 | return function connect(obj, f) { 12 | if (_.isFunction(obj)) { 13 | f = obj; 14 | obj = {}; 15 | } 16 | 17 | var conn = amqp.login(config); 18 | 19 | if (options.amqpError) { 20 | conn.on('error', function defaultAMQPError(err) { 21 | logger.error("AMQP ERROR", err); 22 | throw err; 23 | }); 24 | } 25 | 26 | (obj.queues || []).forEach(function (queueName) { 27 | conn.queue(queueName, { 28 | passive: true 29 | }); 30 | }); 31 | 32 | (obj.exchanges || []).forEach(function (exchangeName) { 33 | conn.exchange(exchangeName, { 34 | passive: true 35 | }); 36 | }); 37 | 38 | conn.connect(f); 39 | }; 40 | }; 41 | -------------------------------------------------------------------------------- /helpers/elasticsearch.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var elasticsearch = require('elasticsearch'); 4 | 5 | module.exports = function (esConfig) { 6 | /** 7 | * Connect to elasticsearch 8 | * @param {Function} f(err, elasticsearchClient) 9 | */ 10 | return function (f) { 11 | // http://www.elasticsearch.org/guide/en/elasticsearch/client/javascript-api/current/host-reference.html 12 | var es = new elasticsearch.Client({ 13 | host: esConfig.host, 14 | log: esConfig.log, 15 | maxRetries: esConfig.maxRetries, 16 | requestTimeout: esConfig.requestTimeout, 17 | deadTimeout: esConfig.deadTimeout 18 | }); 19 | 20 | // try to ping the ES instance to check if everything is fine 21 | es.ping({ 22 | requestTimeout: esConfig.requestTimeout 23 | }, function (err) { 24 | // return both the error (if any) and the es client 25 | f(err, es); 26 | }); 27 | }; 28 | }; 29 | -------------------------------------------------------------------------------- /helpers/logger.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var Winston = require('winston'); 3 | 4 | var winston = new Winston.Logger({ 5 | transports: [ 6 | new Winston.transports.Console({ 7 | colorize: true, 8 | timestamp: true 9 | }) 10 | ], 11 | timestamp: true, 12 | level: 'debug' 13 | }); 14 | 15 | module.exports = winston; 16 | -------------------------------------------------------------------------------- /helpers/validation.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var tv4 = require('tv4'); 4 | tv4.addFormat(require('tv4-formats')); 5 | tv4.addFormat({ 6 | 'enum': function (value, params) { 7 | assert(_.isArray(params.values)); 8 | return _.contains(params.values, value) ? null : '' + params.title + ' should be one of the following values ' + params.values.join(', '); 9 | }, 10 | 'single-level-object': function (obj) { 11 | var isSingleLevel = true; 12 | _.forOwn(obj, function (value /*, key*/ ) { 13 | if (_.isPlainObject(value)) { 14 | isSingleLevel = false; 15 | return false; 16 | } 17 | }); 18 | return isSingleLevel ? null : 'Only single-level object are allowed'; 19 | } 20 | }); 21 | /** 22 | * this method is synchronous 23 | * @param {Mixed} data 24 | * @param {Object} schema json schema 25 | * @param {Function} fallback(data) -> b 26 | * @return {Mixed} either a PrettyError or the data 27 | */ 28 | function validate(data, schema, fallback) { 29 | var err = tv4.validateMultiple(data, schema); 30 | if (!err || _.isObject(err) && err.valid) { 31 | // no error were found, continue 32 | return _.isFunction(fallback) ? fallback(data) : err; 33 | } 34 | 35 | return PrettyError.fromValidation(err); 36 | } 37 | 38 | module.exports = { 39 | validate: validate 40 | }; 41 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "statwarn-monitoring-api", 3 | "version": "0.2.0", 4 | "description": "Statwarn Monitoring API", 5 | "main": "server.js", 6 | "private": true, 7 | "scripts": { 8 | "test-watch": "source ci/.env_test;mocha -w -G -t 5000 -R spec **/*.test.js~node_modules/*", 9 | "ci": "./ci/ci-start.sh" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git@bitbucket.org:statwarn/monitoring-api.git" 14 | }, 15 | "engines": { 16 | "node": "0.10.35" 17 | }, 18 | "keywords": [ 19 | "monitoring", 20 | "api", 21 | "statwarn", 22 | "elasticsearch" 23 | ], 24 | "author": "rbaumier", 25 | "license": "Redsmin", 26 | "dependencies": { 27 | "amqp-dsl": "^2.0.1", 28 | "async": "^0.9.0", 29 | "better-assert": "^1.0.2", 30 | "body-parser": "^1.10.0", 31 | "common-env": "^1.0.0", 32 | "elasticsearch": "^3.0.1", 33 | "express": "^4.10.6", 34 | "filtres": "^0.1.1", 35 | "lodash": "^3.0.1", 36 | "mocha": "^2.0.1", 37 | "moment-range": "^1.0.5", 38 | "node-redis": "^0.1.7", 39 | "redis": "^0.12.1", 40 | "requestretry": "^1.2.1", 41 | "tv4": "^1.1.5", 42 | "tv4-formats": "^1.0.0", 43 | "winston": "^0.9.0" 44 | }, 45 | "devDependencies": { 46 | "chai": "^1.10.0", 47 | "flat": "^1.3.0", 48 | "redis": "^0.12.1", 49 | "redis-info": "^2.0.2", 50 | "request": "^2.49.0", 51 | "supertest": "^0.15.0" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | require(!process.env.TEST ? './bootstrap' : './bootstrap.test'); 3 | var logger = require('./helpers/logger'); 4 | var config = require('./config')(logger); 5 | var template = require('./template')(config); 6 | /** 7 | 8 | * @param {Function} f(app, config, logger, es, amqp) 9 | * @param {Function} fRouteError(err, method, url) 10 | */ 11 | function getConfiguredAPP(f, fRouteError, options) { 12 | var connectAndCheckES = require('./helpers/elasticsearch')(config.elasticsearch); 13 | var connectAndCheckAMQP = require('./helpers/amqp')(config.amqp, logger, options); 14 | 15 | async.parallel({ 16 | es: connectAndCheckES, 17 | amqp: _.partial(connectAndCheckAMQP) 18 | }, function (err, results) { 19 | if (err) { 20 | logger.error(err); 21 | throw err; 22 | } 23 | 24 | logger.info('AMQP & ES ready'); 25 | 26 | var es = results.es; 27 | var amqp = results.amqp; 28 | 29 | amqp.connection.exchange(config.amqp.publish.exchange, { 30 | passive: true, 31 | confirm: true 32 | }, function (exchange) { 33 | amqp.publishExchange = exchange; 34 | // configure the api 35 | var app = require('./api')(config, logger, es, amqp, template, fRouteError); 36 | _.defer(f, app, config, logger, es, amqp); 37 | }); 38 | }); 39 | 40 | } 41 | 42 | function defaultAppHandler(app, config, logger /*, es, amqp */ ) { 43 | app.listen(config.statwarn.monitoring.api.port, function () { 44 | logger.info('API listening on ' + config.statwarn.monitoring.api.port); 45 | }); 46 | } 47 | 48 | function defaultRouteError(err, method, url) { 49 | // @todo track that into stathat/starwarn & co 50 | logger.error(method + url, err); 51 | } 52 | 53 | 54 | if (module.parent === null) { 55 | // the server was directly launched 56 | getConfiguredAPP(defaultAppHandler, defaultRouteError, { 57 | amqpError: true 58 | }); 59 | } else { 60 | // otherwise the server was required by another module 61 | module.exports = getConfiguredAPP; 62 | } 63 | -------------------------------------------------------------------------------- /template/defined/redsmin.template.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-root-object-type.html 3 | // 4 | dynamic_templates: [{ 5 | "tpl_never_analyze": { 6 | "path_match": "data.*", // we don't set "*" because we don't have metadata in redsmin 7 | "mapping": { 8 | // Index this field, so it is searchable, but index the value exactly as specified. Do not analyze it. 9 | "index": "not_analyzed" 10 | } 11 | } 12 | }], 13 | 14 | data_mapping: {} 15 | }; 16 | -------------------------------------------------------------------------------- /template/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var path = require('path'); 3 | 4 | module.exports = function (config) { 5 | assert(_.isString(config.elasticsearch.index.template)); 6 | 7 | return { 8 | base: require('./monitoring.template'), 9 | defined: require(path.resolve(__dirname, config.elasticsearch.index.template)) 10 | }; 11 | }; 12 | -------------------------------------------------------------------------------- /template/monitoring.template.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | "name": "{{INDEX_NAME_PREFIX}}_template", 5 | "template": "{{INDEX_NAME_PREFIX}}-*", 6 | "settings": { 7 | // Can be overriden by ELASTICSEARCH_INDEX_SETTINGS_NUMBER_OF_SHARDS 8 | "number_of_shards": 1, 9 | // Can be overriden by ELASTICSEARCH_INDEX_SETTINGS_NUMBER_OF_REPLICAS 10 | "number_of_replicas": 1 11 | }, 12 | "mappings": { 13 | "{{INDEX_DOCUMENT_TYPE}}": { 14 | "_id": { 15 | // Each document indexed is associated with an id and a type. The _id field can be used to index just the id, and possible also store it. 16 | // By default it is not indexed and not stored (thus, not created). 17 | }, 18 | "_timestamp": { 19 | "enabled": "true", 20 | "type": "long", 21 | "path": "timestamp" 22 | }, 23 | "_all": { 24 | // The idea of the _all field is that it includes the text of one or more other fields within the document indexed. It can come very handy especially for search requests, where we want to execute a search query against the content of a document, without knowing which fields to search on. This comes at the expense of CPU cycles and index size. 25 | "enabled": "false" 26 | }, 27 | "_source": { 28 | // The _source field is an automatically generated field that stores the actual JSON that was used as the indexed document. It is not indexed (searchable), just stored. When executing "fetch" requests, like get or search, the _source field is returned by default. 29 | "enabled": "true" 30 | }, 31 | // http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-dynamic-mapping.html 32 | "dynamic": "strict", 33 | // http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/custom-dynamic-mapping.html 34 | // With dynamic_templates, you can take complete control over the mapping that is generated for newly detected fields. You can even apply a different mapping depending on the field name or datatype. 35 | "dynamic_templates": [], 36 | "properties": { 37 | "timestamp": { 38 | "type": "date", 39 | "index": "not_analyzed" 40 | }, 41 | "data": { 42 | "type": "object", 43 | "dynamic": true, 44 | "properties": {} 45 | }, 46 | "metadata": { 47 | "type": "object", 48 | "dynamic": true, 49 | "properties": {} 50 | } 51 | } 52 | } 53 | } 54 | }; 55 | --------------------------------------------------------------------------------