├── .gitignore
├── LICENSE
├── README.md
├── bin
    └── queryda
├── examples
    ├── cassandra.md
    └── elasticsearch.md
├── jobs
    ├── cqlexample.json
    ├── example.json
    └── mos.json
├── package.json
├── src
    ├── app.js
    ├── main.js
    ├── reporter.js
    ├── reporters
    │   ├── console.js
    │   └── mail.js
    ├── validator.js
    ├── validators
    │   ├── anomalies.js
    │   ├── dropeak.js
    │   ├── eval.js
    │   └── range.js
    └── workers
    │   ├── cassandra.js
    │   └── elastic.js
├── test
    ├── app.js
    ├── reporter-console.js
    ├── reporter-mail.js
    ├── validator.js
    └── worker.js
├── test_cassandra.sh
└── test_elastic.sh


/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2015 Rico Pfaus <ricopfaus@gmail.com>
 4 | Copyright (c) 2015 Lorenzo Mangani <info@qxip.net>
 5 | Copyright (c) 2018 QXIP BV <info@qxip.net>
 6 | 
 7 | Permission is hereby granted, free of charge, to any person obtaining a copy
 8 | of this software and associated documentation files (the "Software"), to deal
 9 | in the Software without restriction, including without limitation the rights
10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 | copies of the Software, and to permit persons to whom the Software is
12 | furnished to do so, subject to the following conditions:
13 | 
14 | The above copyright notice and this permission notice shall be included in all
15 | copies or substantial portions of the Software.
16 | 
17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 | SOFTWARE.
24 | 
25 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # <img width=200 alt="queryda" src="https://user-images.githubusercontent.com/1423657/34468954-46e1d2fe-ef14-11e7-9b1f-279caaaa5b02.png"/>
  2 | 
  3 | <img width=150 src="https://user-images.githubusercontent.com/1423657/34464911-076e9432-ee96-11e7-8c0c-0d893eef2429.png"/>
  4 | 
  5 | 
  6 | **Queryda** loves your data as much as you do! Designed for lightweight, intelligent alarming, it will execute and _"watch"_ **Elasticsearch** and **Cassandra** queries via ```workers```, comparing their results to one or more given expectations via a pipeline of ```validators```. When query results does not match the expectations, a ```reporter``` is notified and can perform any kind of action _(e.g. heat up the coffeemaker via IFTTT before sending an email to your dev team)_.
  7 | 
  8 | For a natively ELK/Elassandra UI integrated and advanced alerting plarform, check out our Kibana App [SENTINL](https://github.com/sirensolutions/sentinl)
  9 | 
 10 | ## Getting started
 11 | 
 12 | #### npm
 13 | Install globally using npm
 14 | ```
 15 | npm install -g queryda
 16 | ```
 17 | #### Manual
 18 | or clone the git repository and install the dependencies locally
 19 | ```
 20 | git clone https://github.com/lmangani/queryda.git
 21 | cd queryda
 22 | npm install
 23 | ```
 24 | 
 25 | 
 26 | ## Usage
 27 | 
 28 | ### Quick Examples
 29 | ##### Elasticsearch
 30 | To see an Elasticsearch example, see [examples/elasticsearch](https://github.com/QXIP/queryda/blob/master/examples/elasticsearch.md)
 31 | 
 32 | ##### Cassandra
 33 | To see a Cassandra example, see [examples/cassandra](https://github.com/QXIP/queryda/blob/master/examples/cassandra.md)
 34 | 
 35 | ---------
 36 | 
 37 | ### Set Alert from Command-Line
 38 | Let's run queryda with the following commandline (or using the *example.json* from the `jobs` dir). 
 39 | ```
 40 | bin/queryda \
 41 | --elasticsearch='{"host":"localhost","port":9200,"index":"monitoring","type":"rum"}' \
 42 | --query='{"range":{"timestamp":{"gt":"2018-01-01T00:00:01","lt":"2018-01-01T23:59:59"}}}' \
 43 | --aggs='{}' \
 44 | --validators='{"range":{"fieldName":"renderTime","min":0,"max":500,"tolerance":4}}' \
 45 | --reporters='{"console":{}}' --debug --name test
 46 | ```
 47 | 
 48 | ### Set Alert from Config
 49 | queryda can also be executed using a self-contained configuration file (see [example.json](jobs/example.json))
 50 | ```
 51 | bin/queryda --configfile /path/to/watcherjob.json
 52 | ```
 53 | 
 54 | ## Configuration
 55 | queryda can be configured either via commandline or using a JSON file _(suggested method via `--configfile` parameter)_. Both ways require to specify option groups with individual settings (e.g. for elasticsearch, for the reporters, for the validator, ..). A set of example JSON files for Cassandra and Elasticsearch can be found in the `jobs`dir.
 56 | 
 57 | The following options are currently available:
 58 | 
 59 | ### *name (required)*
 60 | A name of your choice to identify this job. This will be used by the reporters to identitfy this individual call.
 61 | 
 62 | ## Elasticsearch 
 63 | ### *elasticsearch (elasticsearch only, required)*
 64 | Settings for elasticsearch, expects the following madatory fields:
 65 | - *host*: where to find the elasticsearch host
 66 | - *port*: which port elasticsearch is running on
 67 | - *index*: the index name to send youe query to
 68 | - *type*: the document type to query
 69 | 
 70 | ### *query* (required)
 71 | An elasticsearch query statement. Refer to the [elasticsearch documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current) for details about syntax and features. Should return a result set that contains the supplied *fieldName* to match against.
 72 | 
 73 | ### *aggs* (elasticsearch only, required)
 74 | An elasticsearch aggregation statement. Refer to the [elasticsearch documentation](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current) for details about syntax and features. Should return a result set that contains the supplied *aggName* to match against.
 75 | 
 76 | ## Cassandra
 77 | ### *cassandra (cassandra only, required)*
 78 | Settings for elasticsearch, expects the following madatory fields:
 79 | - *host*: where to find the cassandra host
 80 | - *keyspace*: which to use for queries
 81 | 
 82 | ### *cqlquery* (required)
 83 | A Cassandra query statement. Refer to the [cassandra documentation](http://cassandra.apache.org/doc/latest/cql/) for details about syntax and features. Should return a result set that contains the supplied *fieldName* to match against.
 84 | 
 85 | ### *params* (cassandra only, required)
 86 | A set of Parameters/Variable to be used by the CQL Query.
 87 | 
 88 | ### *validators (required)*
 89 | Validator(s) to compare the query results against. Expects an object with key/value pairs where *key* ist the name of the validator and *value* is the validator-specific configuration. See [Validators](#validators) for more details.
 90 | 
 91 | ### *reporters (required)*
 92 | Reporter(s) to notify about alarms. Expects an object with key/value pairs where *key* ist the name of the reporter and *value* is the reporter-specific configuration. See [Reporters](#reporters) for more details.
 93 | 
 94 | ### *configfile*
 95 | Name of JSON file to read config from. Expects main options as top-level properties (see [example.json](jobs/example.json) for a live example).
 96 | 
 97 | ## Validators
 98 | A Validator takes a query result received from elasticsearch and compares it against a given expectation. This can be as easy as checking if a value equals a given constant or as complex as checking the average of a series of values against an allowed range with an explicit threshold.
 99 | 
100 | ### Available Validators
101 | #### Range
102 | The Range Validator checks a given Field for mix/max boundaries with tolerance factor. 
103 | 
104 | Expects the following mandatory fields:
105 | - *fieldName*: The name of the field in the result set, that is compared against the defined expectation.
106 | - *min*: The minimum allowed value for all values within the query. If a series of values (as defined through the *tolerance* property) in the result is lower than this minimum an alarm is raised and reported.
107 | - *max*: The maxmimum allowed value for all values within the query. If a series of values (as defined through the *tolerance* property) in the result exceed this maximum an alarm is raised and reported.
108 | - *tolerance*: If a queried series of values exceeds either *min* or *max* for *tolerance*+1 times an alarm is raised.
109 | 
110 | ##### Range Example
111 | ```javascript
112 |  "validators": {
113 |     "range": {
114 | 	    "fieldName": "value",
115 | 	    "min": 0,
116 | 	    "max": 500,
117 | 	    "tolerance": 4
118 |     }
119 |   },
120 | 
121 | ```
122 | #### Anomalies
123 | The Anomalies Validator can determine clusters of data and then also identify values which
124 | do not identify with any derived cluster and delcare them outliers.
125 | 
126 | Expects the following mandatory field:
127 | - *fieldName*: The name of the field in the result set, that is tested for series anomalies.
128 | 
129 | ##### Anomalies Example
130 | ```javascript
131 |  "validators": {
132 |     "anomalies": {
133 | 	    "fieldName": "value",
134 | 	    "tolerance": 0
135 | 
136 |     }
137 |   },
138 | 
139 | ```
140 | ### Custom validators
141 | You can create custom validators by creating a new class that extends the `Validator` class (see [RangeValidator](src/validators/range.js) for an example).
142 | 
143 | ## Reporters
144 | By default queryda does nothing more than executing its configured jobs, raising alarms if expectations aren't met. If you want to perform any action in such an alarm case, you have to define a reporter.
145 | 
146 | To put it simple - reporters are notified about alarms, which means a configured expectation isn't met for a given number of times. They can then do helpful things depending on their type like sending an email, creating a ticket in your ticket system, etc.
147 | 
148 | Reporters are defined inside a job's config, you can set either one or multiple of them. Most reporters need a specific configuration that is based on the reporter type.
149 | 
150 | ### Available reporters
151 | 
152 | #### ConsoleReporter
153 | The ConsoleReporter is just meant for demonstration purpose and simply logs a message to the console and has no configuration options.
154 | 
155 | #### MailReporter
156 | The MailReporter sends an email to one (or multiple) given e-mail address(es). It offers the following configuration:
157 | ```javascript
158 | "reporters": {
159 |   "mail": {
160 |     // comma-separated list of target addresses for notification
161 |     "targetAddress": "me@example.com,peng@example.com"
162 |     // number of retry attempts if sending mail fails (defaults to 3)
163 |     "maxRetries": 3
164 |   }
165 | }
166 | ```
167 | 
168 | ### Custom reporters
169 | You can create custom reporters by creating a new class that extends the `Reporter` class (see [ConsoleReporter](src/reporters/console.js) for an example).
170 | 
171 | ## TODO
172 | There's plenty of work to be done in order to make this tool powerful - any help and contribution is appreciated!
173 | 
174 | * [ ] Cleanup, Revamp output for usefulness.
175 | * [ ] Port transform, validate, actions from SENTINL
176 | * [ ] Properly modularize input Workers
177 | * [x] Implement Cassandra Support
178 | * [ ] Implement Clickhouse Support
179 | * [ ] Implement InfluxDB Support
180 | * [ ] Implement GunDB Support
181 |   * [ ] Integrate SENTINL Clustering
182 | * [ ] Implement pseudo-cascading programming as in Kapacitor
183 | 
184 | ## Credits and Acknowledgements
185 | All rights reserved by their respective owners.
186 | 
187 | Original Fork based on Elasticwatch-JS by QXIP, and its [Coffeescript](https://github.com/ryx/elasticwatch) parent.
188 | 
189 | Apache Cassandra, Apache Lucene, Apache, Lucene, Solr, TinkerPop, and Cassandra are trademarks of the Apache Software Foundation or its subsidiaries in Canada, the United States and/or other countries.
190 | 
191 | Elasticsearch and Kibana are trademarks of Elasticsearch BV, registered in the U.S. and in other countries.
192 | 
193 | Elassandra is a trademark of Strapdata SAS.
194 | 
195 | Sentinl is a trademark of QXIP BV and Siren Solutions.
196 | 
197 | 
198 | 


--------------------------------------------------------------------------------
/bin/queryda:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | nodejs src/main.js $*
3 | 


--------------------------------------------------------------------------------
/examples/cassandra.md:
--------------------------------------------------------------------------------
 1 | # Cassandra Watcher
 2 | 
 3 | ## Example
 4 | 
 5 | ### Dataset
 6 | Create Keyspace:
 7 | ```
 8 | cqlsh -e "CREATE KEYSPACE test WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor': 1};" --cqlversion="3.4.4"
 9 | ```
10 | Create Table:
11 | ```
12 | cqlsh -e "CREATE TABLE test.TEST (ID TEXT, NAME TEXT, value TEXT, LAST_MODIFIED_DATE TIMESTAMP, PRIMARY KEY (ID));" --cqlversion="3.4.4"
13 | ```
14 | Insert Sample Data:
15 | ```
16 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('2', 'elephant',  '488', toTimestamp(now()));" --cqlversion="3.4.4" && sleep 1;
17 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('3', 'elephant',  '598', toTimestamp(now()));" --cqlversion="3.4.4" && sleep 1;
18 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('4', 'elephant',  '999', toTimestamp(now()));" --cqlversion="3.4.4" && sleep 1;
19 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('5', 'elephant',  '566', toTimestamp(now()));" --cqlversion="3.4.4" && sleep 1;
20 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('6', 'elephant',  '521', toTimestamp(now()));" --cqlversion="3.4.4" && sleep 1;
21 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('7', 'elephant',  '590', toTimestamp(now()));" --cqlversion="3.4.4" && sleep 1;
22 | ```
23 | 
24 | ### Alert from Command-Line + Config File
25 | Once created, execute queryda with the following commandline using the *cqlexample.json* from the `jobs` dir. 
26 | ```
27 | bin/queryda --configfile="../jobs/cqlexample.json"
28 | ```
29 | ###### Output
30 | ```
31 | ConsoleReporter.notify: 'SimpleJob-5m' raised alarm: ALARM_VALIDATION_FAILED: 'value' outside range '0-600' for '1' consecutive times: '999'
32 | ```
33 | 
34 | #### Example Configuration
35 | The following examples illustrates a time-bound
36 | ```
37 | {
38 |   "name": "SimpleJob-5m",
39 |   "info": "This job simply queries some values and compares them to a given min and max range",
40 |   "cassandra": "127.0.0.1",
41 |   "cqlquery": "SELECT value FROM test.TEST WHERE LAST_MODIFIED_DATE >= '2018-01-01 00:01+0000' AND LAST_MODIFIED_DATE <= toTimestamp(now()) LIMIT 100 ALLOW FILTERING;",
42 |   "params": "null",
43 |   "validators": {
44 |     "range" : {
45 | 	    "fieldName": "value",
46 | 	    "min": 0,
47 | 	    "max": 600,
48 | 	    "tolerance": 1
49 |     }
50 |   },
51 |   "reporters": {
52 |     "console": {}
53 |   }
54 | }
55 | ```
56 | 
57 | ### Custom Time-Range Function
58 | To allow simple time-range queries in Cassandra 3.x, the following custom Function can be used. Note custom functions requires ```enable_user_defined_functions=true``` in cassandra.yaml ahead of usage.
59 | ```
60 | CREATE FUNCTION IF NOT EXISTS toTimestampLast(minutes int) 
61 |   CALLED ON NULL INPUT 
62 |   RETURNS timestamp
63 |   LANGUAGE java AS '
64 |     long now = System.currentTimeMillis();
65 |     if (minutes == null)
66 |       return new Date(now);
67 |     return new Date(now - (minutes.intValue() * 60 * 1000));
68 |   ';
69 | ```
70 | Usage:
71 | ```
72 | SELECT value FROM test.TEST WHERE LAST_MODIFIED_DATE >= toTimestampLast(60) AND LAST_MODIFIED_DATE <= toTimestamp(now()) LIMIT 100 ALLOW FILTERING;
73 | ```
74 | 


--------------------------------------------------------------------------------
/examples/elasticsearch.md:
--------------------------------------------------------------------------------
 1 | # Elastic Watcher
 2 | 
 3 | ## Example
 4 | 
 5 | ### Dataset
 6 | Once installed, create some fictional data in our elasticsearch
 7 | ```bash
 8 | curl -s -XPUT 'http://localhost:9200/monitoring/rum/1' -d '{"requestTime":43,"responseTime":224,"renderTime":568,"timestamp":"2015-03-06T11:47:34"}'
 9 | curl -s -XPUT 'http://localhost:9200/monitoring/rum/2' -d '{"requestTime":49,"responseTime":312,"renderTime":619,"timestamp":"2015-03-06T12:02:34"}'
10 | curl -s -XPUT 'http://localhost:9200/monitoring/rum/3' -d '{"requestTime":41,"responseTime":275,"renderTime":597,"timestamp":"2015-03-06T12:17:34"}'
11 | curl -s -XPUT 'http://localhost:9200/monitoring/rum/4' -d '{"requestTime":42,"responseTime":301,"renderTime":542,"timestamp":"2015-03-06T12:32:34"}'
12 | curl -s -XPUT 'http://localhost:9200/monitoring/rum/5' -d '{"requestTime":48,"responseTime":308,"renderTime":604,"timestamp":"2015-03-06T12:47:34"}'
13 | curl -s -XPUT 'http://localhost:9200/monitoring/rum/6' -d '{"requestTime":43,"responseTime":256,"renderTime":531,"timestamp":"2015-03-06T13:02:34"}'
14 | ```
15 | ### Alert from Command-Line
16 | Once created, execute queryda with the following commandline (or using the *example.json* from the `jobs` dir). 
17 | ```
18 | bin/queryda \
19 | --elasticsearch='{"host":"localhost","port":9200,"index":"monitoring","type":"rum"}' \
20 | --query='{"range":{"timestamp":{"gt":"2015-03-06T12:00:00","lt":"2015-03-07T00:00:00"}}}' \
21 | --aggs='{}' \
22 | --validators='{"range":{"fieldName":"renderTime","min":0,"max":500,"tolerance":4}}' \
23 | --reporters='{"console":{}}' --debug --name test
24 | ```
25 | 
26 | ### Alert from Config
27 | queryda can also be executed using a self-contained configuration file (see [example.json](jobs/example.json))
28 | ```
29 | bin/queryda --configfile /path/to/watcherjob.json
30 | ```
31 | 


--------------------------------------------------------------------------------
/jobs/cqlexample.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "SimpleJob-5m",
 3 |   "info": "This job simply queries some values and compares them to a given min and max range",
 4 |   "cassandra": "127.0.0.1",
 5 |   "cqlquery": "SELECT value from test.TEST",
 6 |   "params": "test",
 7 |   "validators": {
 8 |     "range" : {
 9 | 	    "fieldName": "value",
10 | 	    "min": 0,
11 | 	    "max": 600,
12 | 	    "tolerance": 1
13 |     },
14 |     "eval" : {
15 | 	"fieldName": "value",
16 | 	"exp": "data[.value > 600].value",
17 | 	"tolerance": 0
18 |     }
19 |   },
20 |   "reporters": {
21 |     "console": {}
22 |   }
23 | }
24 | 


--------------------------------------------------------------------------------
/jobs/example.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "SimpleJob-5m",
 3 |   "info": "This job simply queries some values and compares them to a given min and max range",
 4 |   "elasticsearch": {
 5 |     "host": "localhost",
 6 |     "port": "9200",
 7 |     "index": "monitoring",
 8 |     "type": "rum"
 9 |   },
10 |   "query": {
11 |       "range": {
12 |             "timestamp": {
13 |               "gte": "2017-01-06T12:00:00",
14 |               "lte": "2017-12-07T00:00:00"
15 |             }
16 |       }
17 |   },
18 |   "aggs": {},
19 |   "validators": {
20 |     "range" : {
21 | 	    "fieldName": "renderTime",
22 | 	    "min": 0,
23 | 	    "max": 500,
24 | 	    "tolerance": 1
25 |     },
26 |     "anomalies" : {
27 | 	    "fieldName": "renderTime",
28 | 	    "tolerance": 2
29 |     }
30 |   },
31 |   "reporters": {
32 |     "console": {}
33 |   }
34 | }
35 | 


--------------------------------------------------------------------------------
/jobs/mos.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "SimpleJob-5m",
 3 |   "info": "This job simply queries some values and compares them to a given min and max range",
 4 |   "elasticsearch": {
 5 |     "host": "localhost",
 6 |     "port": "9200",
 7 |     "index": "mos-*",
 8 |     "type": "mos"
 9 |   },
10 |   "query": {
11 |     "filtered": {
12 |       "query": {
13 |         "query_string": {
14 |           "query": "mos:*",
15 |           "analyze_wildcard": true
16 |         }
17 |       },
18 |       "filter": {
19 |         "range" : {
20 |           "@timestamp" : {
21 |             "gt" : "now-5m",
22 |             "lt" : "now"
23 |           }
24 |         }
25 |       }
26 |     }
27 |   },
28 |   "aggs": {},
29 |   "validators": {
30 |     "range" : {
31 | 	    "fieldName": "mos",
32 | 	    "min": 3,
33 | 	    "max": 5,
34 | 	    "tolerance": 1
35 |     }
36 |   },
37 |   "reporters": {
38 |     "console": {}
39 |   }
40 | }
41 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "queryda",
 3 |   "version": "1.0.8",
 4 |   "description": "watch/report/alert tool for elasticsearch",
 5 |   "author": "Lorenzo Mangani <lorenzo.mangani@gmail.com>",
 6 |   "keywords": [
 7 |     "elasticsearch",
 8 |     "cassandra",
 9 |     "elassandra",
10 |     "alarming",
11 |     "alerting",
12 |     "monitoring",
13 |     "watcher",
14 |     "notification"
15 |   ],
16 |   "bin": {
17 |     "queryda": "./bin/queryda"
18 |   },
19 |   "repository": {
20 |     "type": "git",
21 |     "url": "https://github.com/lmangani/queryda"
22 |   },
23 |   "dependencies": {
24 |     "cassandra-driver": "^3.3.0",
25 |     "jexl": "^1.1.4",
26 |     "loglevel": "^1.2.0",
27 |     "url": "*",
28 |     "yargs": "^3.4.5"
29 |   },
30 |   "devDependencies": {
31 |     "chai": "^2.1.1",
32 |     "mocha": "^2.2.1",
33 |     "mockery": "^1.4.0"
34 |   },
35 |   "engines": {
36 |     "node": ">=0.8.0"
37 |   },
38 |   "license": "MIT",
39 |   "scripts": {
40 |     "test": "./test_elastic.sh && test_cassandra.sh",
41 |     "test_elastic": "./test_elastic.sh",
42 |     "test_cassandra": "./test_cassandra.sh"
43 |   }
44 | }
45 | 


--------------------------------------------------------------------------------
/src/app.js:
--------------------------------------------------------------------------------
  1 | (function() {
  2 |   var App, Worker, cqlWorker, log,
  3 |     bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; };
  4 | 
  5 |   log = require("loglevel");
  6 | 
  7 |   Worker = require("./workers/elastic.js");
  8 |   cqlWorker = require("./workers/cassandra.js");
  9 | 
 10 |   /**
 11 |    * The main application logic and entry point. Reads args, sets things up,
 12 |    * runs workers.
 13 |    *
 14 |    * @class  App
 15 |    */
 16 | 
 17 |   module.exports = App = (function() {
 18 | 
 19 |     /**
 20 |      * Create a new App based on the given configuration.
 21 |      *
 22 |      * @constructor
 23 |      * @param  config  {Object}  object with configuration options.
 24 |      */
 25 |     function App(config1) {
 26 |       var cfg, j, len, ref, ref1, reporter, reporterName, s, ValidatorName, validator;
 27 |       this.config = config1;
 28 |       this.handleAlarm = bind(this.handleAlarm, this);
 29 |       log.debug("App.constructor: creating app", this.config);
 30 | 
 31 |       this.reporters = [];
 32 |       ref1 = this.config.reporters;
 33 |       for (reporterName in ref1) {
 34 |         cfg = ref1[reporterName];
 35 |         log.debug("App.constructor: creating reporter '" + reporterName + "'");
 36 |         reporter = App.createReporter(reporterName, cfg);
 37 |         if (reporter) {
 38 |           this.reporters.push(reporter);
 39 |         }
 40 |       }
 41 | 
 42 |       this.validators = [];
 43 |       ref1 = this.config.validators;
 44 |       for (var validatorName in ref1) {
 45 |         cfg = ref1[validatorName];
 46 |         log.debug("App.constructor: creating validator '" + validatorName + "'");
 47 |         validator = App.createValidator(validatorName, cfg);
 48 |         if (validator) {
 49 |           this.validators.push(validator);
 50 |         }
 51 |       }
 52 | 
 53 | 
 54 |       if(this.config.cqlquery) {
 55 | 	      ref = ["name", "cassandra", "cqlquery", "params", "reporters", "validators"];
 56 | 	      for (j = 0, len = ref.length; j < len; j++) {
 57 | 	        s = ref[j];
 58 | 	        if (!this.config[s]) {
 59 | 	          throw new Error("App.constructor: CQL config." + s + " missing");
 60 | 	        }
 61 | 	      }
 62 | 	      this.worker = App.createCqlWorker(this.config.name, this.config.cassandra, this.config.cqlquery, this.config.params, this.validators);
 63 | 	      if (this.worker) {
 64 | 	        this.worker.on("alarm", this.handleAlarm);
 65 | 	        this.worker.start();
 66 | 	      } else {
 67 | 	        throw new Error("App.constructor: CQL worker creation failed");
 68 | 	      }
 69 | 
 70 |       } else {
 71 | 	      ref = ["name", "elasticsearch", "query", "aggs", "reporters", "validators"];
 72 | 	      for (j = 0, len = ref.length; j < len; j++) {
 73 | 	        s = ref[j];
 74 | 	        if (!this.config[s]) {
 75 | 	          throw new Error("App.constructor: ES config." + s + " missing");
 76 | 	        }
 77 | 	      }
 78 | 	      this.worker = App.createElasticWorker(this.config.name, this.config.elasticsearch, this.config.query, this.config.aggs, this.validators);
 79 | 	      if (this.worker) {
 80 | 	        this.worker.on("alarm", this.handleAlarm);
 81 | 	        this.worker.start();
 82 | 	      } else {
 83 | 	        throw new Error("App.constructor: Elastic worker creation failed");
 84 | 	      }
 85 | 
 86 |       }
 87 | 
 88 |     }
 89 | 
 90 | 
 91 |     /**
 92 |      * Instantiate Elastic Worker according to a given configuration.
 93 |      *
 94 |      * @method createElasticWorker
 95 |      * @static
 96 |      * @param  name                  {String}    worker name/id
 97 |      * @param  elasticsearchConfig   {Object}    elasticsearch config (host/port/index/type)
 98 |      * @param  query                 {Object}    elasticsearch query object
 99 |      * @param  aggs                  {Object}    elasticsearch aggs object
100 |      * @param  validator             {Validator} validator object to be passed to Worker
101 |      */
102 | 
103 |     App.createElasticWorker = function(name, elasticsearchConfig, query, aggs, validator) {
104 |       var e, error;
105 |       if (!name || !elasticsearchConfig || !query || !aggs || !validator) {
106 |         log.error("App.createElasticWorker: invalid number of options");
107 |         return null;
108 |       }
109 |       try {
110 |         return new Worker(name, elasticsearchConfig.host, elasticsearchConfig.port, "/" + elasticsearchConfig.index + "/" + elasticsearchConfig.type, query, aggs, validator);
111 |       } catch (error) {
112 |         e = error;
113 |         log.error("ERROR: worker creation failed: ",e);
114 |         return null;
115 |       }
116 |     };
117 | 
118 | 
119 |     /**
120 |      * Instantiate CQL Worker according to a given configuration.
121 |      *
122 |      * @method createCqlWorker
123 |      * @static
124 |      * @param  name                  {String}    worker name/id
125 |      * @param  cassandraConfig       {Object}    elasticsearch config (host/keyspace)
126 |      * @param  cqlquery              {Object}    elasticsearch query object
127 |      * @param  params                {Object}    elasticsearch params object
128 |      * @param  validator             {Validator} validator object to be passed to Worker
129 |      */
130 | 
131 |     App.createCqlWorker = function(name, cassandraConfig, cqlquery, params, validator) {
132 |       var e, error;
133 |       if (!name || !cassandraConfig || !cqlquery || !validator) {
134 |         log.error("App.createCqlWorker: invalid number of options");
135 |         return null;
136 |       }
137 |       try {
138 |         return new cqlWorker(name, cassandraConfig, cqlquery, params, validator);
139 |       } catch (error) {
140 |         e = error;
141 |         log.error("ERROR: worker creation failed: ",e);
142 |         return null;
143 |       }
144 |     };
145 | 
146 | 
147 |     /**
148 |      * Instantiate a Reporter according to a given configuration.
149 |      *
150 |      * @method createReporter
151 |      * @static
152 |      * @param  name    {String} module name of reporter to create
153 |      * @param  config  {Object} hash with reporter configuration object
154 |      */
155 | 
156 |     App.createReporter = function(name, config) {
157 |       var e, error, o, r;
158 |       log.debug("App.createReporter: creating reporter: " + name + " ", config);
159 |       try {
160 |         r = require("./reporters/" + name);
161 |         o = new r(config);
162 |         return o;
163 |       } catch (error) {
164 |         e = error;
165 |         log.error("ERROR: failed to instantiate reporter '" + name + "': " + e.message, r);
166 |         return null;
167 |       }
168 |     };
169 | 
170 | 
171 |     /**
172 |      * Instantiate a Validator according to a given configuration.
173 |      * @FIXME: currently there is only one validator but in the future there will be more different types
174 |      *
175 |      * @method createValidator
176 |      * @static
177 |      * @param  name    {String} module name of validator to create
178 |      * @param  config  {Object} hash with validator configuration object
179 |      */
180 | 
181 |     App.createValidator = function(name, config) {
182 |       var e, error, o, r;
183 |       log.debug("App.createValidator: creating validator: " + name + " ", config);
184 |       try {
185 |         r = require("./validators/" + name);
186 |         o = new r(config);
187 |         return o;
188 |       } catch (error) {
189 |         e = error;
190 |         log.error("ERROR: failed to instantiate validator '" + name + "': " + e.message, r);
191 |         return null;
192 |       }
193 |     };
194 | 
195 | /*
196 |     App.createValidator = function(name, config) {
197 |       var e, error, o;
198 |       log.debug("App.createValidator: creating validator: " + name + " ", config);
199 |       try {
200 |         o = new Validator(config.fieldName, config.min, config.max, config.tolerance);
201 |         return o;
202 |       } catch (error) {
203 |         e = error;
204 |         log.error("ERROR: failed to instantiate validator '" + name + "': " + e.message, o);
205 |         return null;
206 |       }
207 |     };
208 | */
209 | 
210 | 
211 |     /**
212 |      * Handle alarm event sent by a worker. Notifies all exitsing reporters about
213 |      * a given event.
214 |      *
215 |      * @method handleAlarm
216 |      * @param  event {Object}  the event object passed to alarm event
217 |      */
218 | 
219 |     App.prototype.handleAlarm = function(message, data) {
220 |       var i, j, len, ref, reporter, results;
221 |       log.debug("App.handleAlarm: " + message, data);
222 |       ref = this.reporters;
223 |       results = [];
224 |       for (i = j = 0, len = ref.length; j < len; i = ++j) {
225 |         reporter = ref[i];
226 |         results.push(reporter.notify(message, data));
227 |       }
228 |       return results;
229 |     };
230 | 
231 |     return App;
232 | 
233 |   })();
234 | 
235 | }).call(this);
236 | 


--------------------------------------------------------------------------------
/src/main.js:
--------------------------------------------------------------------------------
  1 | (function() {
  2 |   var App, argv, e, error, error1, log, opts, yargs;
  3 | 
  4 |   App = require("./app");
  5 | 
  6 |   log = require("loglevel");
  7 | 
  8 |   yargs = require("yargs");
  9 | 
 10 |   argv = yargs.usage("Usage: $0 --name=[name] --[elasticsearch,query,aggs,cassandra,cqlquery,params,reporters,validators]={...} or --config=[config]").epilog("Queryda by Lorenzo Mangani | (c) 2018 QXIP BV | <info@qxip.net>").version(function() {
 11 |     return require("../package.json").version;
 12 |   }).option("name", {
 13 |     describe: "identifier for this Job (will be included in reports)",
 14 |     type: "string"
 15 |   }).option("elasticsearch", {
 16 |     describe: "object with elasticsearch settings [host|port|index|type]",
 17 |     type: "string"
 18 |   }).option("query", {
 19 |     describe: "elasticsearch query (e.g. {\"match\":\"*\"})",
 20 |     type: "string"
 21 |   }).option("aggs", {
 22 |     describe: "elasticsearch aggs query (e.g. {\"aggs\":\"*\"})",
 23 |     type: "string"
 24 | 
 25 |   }).option("cassandra", {
 26 |     describe: "object with cassandra settings [host]",
 27 |     type: "string"
 28 |   }).option("cqlquery", {
 29 |     describe: "cql query",
 30 |     type: "string"
 31 |   }).option("params", {
 32 |     describe: "cql params",
 33 |     type: "string"
 34 | 
 35 |   }).option("reporters", {
 36 |     describe: "reporters to notify about alarms (as hash with name:config)",
 37 |     type: "string"
 38 |   }).option("validators", {
 39 |     describe: "validators for checking expectation (as hash with name:config)",
 40 |     type: "string"
 41 |   }).option("configfile", {
 42 |     describe: "optional file with JSON data that supplies all options [elasticsearch|query|aggs|validators|reporters]",
 43 |     type: "string"
 44 |   }).option("debug", {
 45 |     describe: "show additional output (for debugging only)",
 46 |     type: "boolean"
 47 |   }).argv;
 48 | 
 49 |   if (argv.configfile) {
 50 |     try {
 51 |       opts = require(argv.configfile);
 52 |     } catch (error) {
 53 |       e = error;
 54 |       log.error("ERROR: failed loading configfile: " + e.message);
 55 |       process.exitCode = 10;
 56 |     }
 57 |   } else {
 58 |     if (!(argv.name || (argv.elasticsearch || argv.query || argv.aggs) || (argv.cassandra || argv.cqlquery) || argv.reporters || argv.validators)) {
 59 |       log.error(yargs.help());
 60 |       process.exitCode = 11;
 61 |     } else {
 62 | 
 63 |       if (argv.elasticsearch){
 64 | 	      try {
 65 | 	        opts = {
 66 | 	          name: argv.name,
 67 | 	          elasticsearch: JSON.parse(argv.elasticsearch),
 68 | 	          query: JSON.parse(argv.query),
 69 | 	          aggs: JSON.parse(argv.aggs),
 70 | 	          validators: JSON.parse(argv.validators),
 71 | 	          reporters: JSON.parse(argv.reporters)
 72 | 	        };
 73 | 	      } catch (error1) {
 74 | 	        e = error1;
 75 | 	        log.error("ERROR: failed parsing elastic commandline options: " + e.message);
 76 | 	        process.exitCode = 12;
 77 | 	      }
 78 |       } else if (argv.cassandra){
 79 | 	      try {
 80 | 	        opts = {
 81 | 	          name: argv.name,
 82 | 	          cassandra: argv.cassandra || "127.0.0.1",
 83 | 	          cqlquery: argv.cqlquery,
 84 | 	          params: argv.params || "",
 85 | 	          validators: JSON.parse(argv.validators),
 86 | 	          reporters: JSON.parse(argv.reporters)
 87 | 	        };
 88 | 	      } catch (error1) {
 89 | 	        e = error1;
 90 | 	        log.error("ERROR: failed parsing cql commandline options: " + e.message);
 91 | 	        process.exitCode = 12;
 92 | 	      }
 93 |       }
 94 |     }
 95 |   }
 96 | 
 97 |   if (process.exitCode > 9) {
 98 |     process.exit();
 99 |   }
100 | 
101 |   log.setLevel(argv.debug ? 1 : 4);
102 | 
103 |   new App(opts);
104 | 
105 | }).call(this);
106 | 


--------------------------------------------------------------------------------
/src/reporter.js:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * The Reporter is an abstract base class that takes information from a
 3 |  * Worker and can do anything with that data. Actual implementations might
 4 |  * do things as e.g. send an email or create a ticket.
 5 |  *
 6 |  * @class  Reporter
 7 |  */
 8 | 
 9 | (function() {
10 |   var Reporter;
11 | 
12 |   module.exports = Reporter = (function() {
13 | 
14 |     /**
15 |      * Create new Reporter with the given configuration object
16 |      *
17 |      * @constructor
18 |      */
19 |     function Reporter(config) {
20 |       this.config = config;
21 |     }
22 | 
23 | 
24 |     /**
25 |      * Notify the reporter about available information
26 |      *
27 |      * @method notify
28 |      * @param  type      {String}  type of this notification
29 |      * @param  message   {String}  human-readable message that describes the incident
30 |      * @param  data      {Object}  any kind of data to pass along with the ntofication (might depend on type)
31 |      */
32 | 
33 |     Reporter.prototype.notify = function(message, data) {};
34 | 
35 |     return Reporter;
36 | 
37 |   })();
38 | 
39 | }).call(this);
40 | 


--------------------------------------------------------------------------------
/src/reporters/console.js:
--------------------------------------------------------------------------------
 1 | (function() {
 2 |   var ConsoleReporter, Reporter, log,
 3 |     extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
 4 |     hasProp = {}.hasOwnProperty;
 5 | 
 6 |   log = require("loglevel");
 7 | 
 8 |   Reporter = require("../reporter");
 9 | 
10 | 
11 |   /**
12 |    * A Reporter that logs an error message to the console. As simple as possible,
13 |    * but should illustrate the basic idea of what a reporter is all about.
14 |    *
15 |    * @class    ConsoleReporter
16 |    * @extends  Reporter
17 |    */
18 | 
19 |   module.exports = ConsoleReporter = (function(superClass) {
20 |     extend(ConsoleReporter, superClass);
21 | 
22 |     function ConsoleReporter(config) {
23 |       this.config = config;
24 |       log.debug("ConsoleReporter.constructor: creating new instance", this.config);
25 |     }
26 | 
27 |     ConsoleReporter.prototype.notify = function(message, data) {
28 |       return log.error("ConsoleReporter.notify: '" + data.name + "' raised alarm: " + message);
29 |     };
30 | 
31 |     return ConsoleReporter;
32 | 
33 |   })(Reporter);
34 | 
35 | }).call(this);
36 | 


--------------------------------------------------------------------------------
/src/reporters/mail.js:
--------------------------------------------------------------------------------
 1 | (function() {
 2 |   var MailReporter, Reporter, exec, log,
 3 |     extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
 4 |     hasProp = {}.hasOwnProperty;
 5 | 
 6 |   log = require("loglevel");
 7 | 
 8 |   exec = require('child_process').exec;
 9 | 
10 |   Reporter = require("../reporter");
11 | 
12 | 
13 |   /**
14 |    * A Reporter that sends an e-mail to a given address, using the system's mail
15 |    * commandline client.
16 |    *
17 |    * @class    MailReporter
18 |    * @extends  Reporter
19 |    */
20 | 
21 |   module.exports = MailReporter = (function(superClass) {
22 |     extend(MailReporter, superClass);
23 | 
24 |     function MailReporter(config) {
25 |       this.config = config != null ? config : {};
26 |       this.maxRetries = this.config.maxRetries || 3;
27 |       this.retryAttempt = 0;
28 |       log.debug("MailReporter.constructor: creating new instance", this.config);
29 |       if (!this.config.targetAddress) {
30 |         log.error("ERROR: mail reporter requires 'targetAddress' in configuration");
31 |       }
32 |     }
33 | 
34 | 
35 |     /**
36 |      * Send a mail (using the system's "mail" commandline tool).
37 |      *
38 |      * @method sendMail
39 |      * @param  target      {String}    e-mail address (or comma-separated list of addresses) to send mail to
40 |      * @param  subject     {String}    mail subject
41 |      * @param  body        {String}    message body
42 |      * @param  onSuccess   {Function}  success callback
43 |      * @param  onError     {Function}  error callback
44 |      */
45 | 
46 |     MailReporter.prototype.sendMail = function(target, subject, body, onSuccess, onError) {
47 |       var child;
48 |       if (onSuccess == null) {
49 |         onSuccess = (function() {});
50 |       }
51 |       if (onError == null) {
52 |         onError = (function() {});
53 |       }
54 |       return child = exec("echo \"" + body + "\" | mail -s \"" + subject + "\" " + target, function(error, stdout, stderr) {
55 |         if (error !== null) {
56 |           return onError(error);
57 |         } else {
58 |           return onSuccess();
59 |         }
60 |       });
61 |     };
62 | 
63 | 
64 |     /**
65 |      * Send notification to this reporter.
66 |      */
67 | 
68 |     MailReporter.prototype.notify = function(message, data) {
69 |       log.debug("MailReporter.notify: '" + data.name + "' raised alarm: " + message);
70 |       return this.sendMail(this.config.targetAddress, "[queryda] " + data.name + " raised alarm", "Hi buddy, \n\nalarm message was: " + message + "\n\nCheers,\nyour Queryda", function() {}, (function(_this) {
71 |         return function(error) {
72 |           log.error("ERROR: mail delivery failed: " + error);
73 |           if (_this.retryAttempt < _this.maxRetries) {
74 |             _this.retryAttempt++;
75 |             return _this.notfiy(message, data);
76 |           } else {
77 |             log.error("ERROR: mail delivery failed " + _this.maxRetries + " times");
78 |             return _this.retryAttempt = 0;
79 |           }
80 |         };
81 |       })(this));
82 |     };
83 | 
84 |     return MailReporter;
85 | 
86 |   })(Reporter);
87 | 
88 | }).call(this);
89 | 


--------------------------------------------------------------------------------
/src/validator.js:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * The Validator is an abstract base class that takes information from a
 3 |  * Worker and can do anything with that data. Actual implementations might
 4 |  * do things as e.g. compare series to thresholds and tolerance factors.
 5 |  *
 6 |  * @class  Validator
 7 |  */
 8 | 
 9 | (function() {
10 |   var Validator;
11 | 
12 |   module.exports = Validator = (function() {
13 | 
14 |     /**
15 |      * Create new Validator with the given configuration object
16 |      *
17 |      * @constructor
18 |      */
19 |     function Validator(config) {
20 |       this.config = config;
21 |     }
22 | 
23 | 
24 |     /**
25 |      * Notify the validator about available information
26 |      *
27 |      * @method notify
28 |      * @param  type      {String}  type of this notification
29 |      * @param  message   {String}  human-readable message that describes the incident
30 |      * @param  data      {Object}  any kind of data to pass along with the ntofication (might depend on type)
31 |      */
32 | 
33 |     Validator.prototype.notify = function(message, data) {};
34 | 
35 |     return Validator;
36 | 
37 |   })();
38 | 
39 | }).call(this);
40 | 


--------------------------------------------------------------------------------
/src/validators/anomalies.js:
--------------------------------------------------------------------------------
  1 | (function() {
  2 |   var AnomaliesValidator, Validator, log;
  3 |   var bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; };
  4 |   var extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
  5 |   var hasProp = {}.hasOwnProperty;
  6 | 
  7 |   log = require("loglevel");
  8 | 
  9 |   Validator = require("../validator");
 10 | 
 11 |   /**
 12 |    * The AnomaliesValidator takes an elasticsearch query result and identify 
 13 |    * values which do not identify with any derived cluster and delcare them 
 14 |    * outliers.
 15 |    *
 16 |    * @class   AnomaliesValidator
 17 |    * @extends  Validator
 18 |    */
 19 | 
 20 |   module.exports = AnomaliesValidator = (function(superClass) {
 21 |     extend(AnomaliesValidator, superClass);
 22 | 
 23 |     /**
 24 |      * Create a new AnomaliesValidator with the given options.
 25 |      * @constructor
 26 |      * @param  fieldName {String}  name of the result field (key) to use as comaprison value
 27 |      * @param  tolerance {int}     maximum allowed number of consecutive values that do not match the expectation
 28 |      */
 29 | 
 30 |     function AnomaliesValidator(config) {
 31 |       this.config = config != null ? config : {};
 32 |       this.fieldName = this.config.fieldName;
 33 |       if (!this.config.tolerance) {
 34 | 		this.tolerance = 0; 
 35 |       } else { 
 36 | 	     	this.tolerance = this.config.tolerance;
 37 |       }
 38 |       this.validate = bind(this.validate, this);
 39 |       this.fails = [];
 40 |       if (!this.fieldName || this.min === null || this.max === null || this.tolerance === null) {
 41 |         throw new Error("invalid number of options");
 42 |       }
 43 |     }
 44 | 
 45 | 
 46 |     /*
 47 |      * Validate the given series for anomalies and return outliers
 48 |      *
 49 |      * @method clustering.nearness
 50 |      * @param  obj  {Object}  series object
 51 |      */
 52 | 
 53 |      //Detect Cluster Types
 54 |      clustering = {};
 55 |      clustering.differentiateGroups = function (obj){
 56 | 	var sortList = [];
 57 | 	var diffs = {};
 58 | 	for(var i in obj){
 59 | 		sortList.push(obj[i]);
 60 | 	}
 61 | 	sortList.sort(function(a,b){
 62 |    		return a - b;
 63 | 	});
 64 | 	for(var i=0; i<sortList.length-1;i++){
 65 | 		var value1 = sortList[i];
 66 | 		var value2 = sortList[i+1];
 67 | 		var diff = Math.abs(Math.abs(value1) - Math.abs(value2));
 68 | 		if(diffs[diff.toString()] == undefined){
 69 | 			diffs[diff.toString()] = 1;
 70 | 		}else{
 71 | 			diffs[diff.toString()] = diffs[diff.toString()] + 1;
 72 | 		}
 73 | 	}
 74 | 	return diffs;
 75 |      }
 76 | 
 77 |      //Form clusters as identifiable groups and outliers
 78 |      clustering.nearness = function (obj){
 79 | 	var clusterBounds = clustering.differentiateGroups(obj);
 80 | 	var bound = 0; 
 81 | 	var weight = 0;
 82 | 	for(var i in clusterBounds){
 83 | 		if(clusterBounds[i] >= weight && Number(i) > bound){
 84 | 			bound = Number(i);
 85 | 			weight = clusterBounds[i];
 86 | 		}
 87 | 	}
 88 | 	
 89 | 	var sortList = [];
 90 | 	var hashList = {};
 91 | 	var nameList = {};
 92 | 	var out = [];
 93 | 	for(var i in obj){
 94 | 		nameList[i] = "outlier_"+obj[i];
 95 | 		sortList.push(obj[i]);
 96 | 		if(hashList[obj[i]] == undefined){
 97 | 			hashList[obj[i]] = [i];
 98 | 		}else{
 99 | 			hashList[obj[i]].push(i);
100 | 		}
101 | 	}
102 | 	sortList.sort(function(a,b){
103 |    		return a - b;
104 | 	});
105 | 	var activeCluster = false;
106 | 	var currentCluster;
107 | 	for(var i=0; i<sortList.length-1;i++){
108 | 		var value1 = sortList[i];
109 | 		var value2 = sortList[i+1];
110 | 		var diff = Math.abs(Math.abs(value1) - Math.abs(value2));
111 | 		if(diff <= bound){
112 | 			if(activeCluster == false){
113 | 				currentCluster = {};
114 | 				out.push(currentCluster);
115 | 				activeCluster = true;
116 | 			}
117 | 			
118 | 			var vals = hashList[value1].concat(hashList[value2]);
119 | 			for(var n in vals){
120 | 				var value = vals[n];
121 | 				delete nameList[value];
122 | 				currentCluster[value] = diff;
123 | 			}
124 | 		}else{
125 | 			
126 | 			activeCluster = false;
127 | 		}
128 | 	}
129 | 	return {"clusters":out, "outliers": nameList};
130 |      }
131 | 
132 | 
133 |     /*
134 |      * Validate the given elasticsearch query result for series anomalies.
135 |      *
136 |      * @method validate
137 |      * @param  data  {Object}  elasticsearch query result
138 |      */
139 | 
140 |     AnomaliesValidator.prototype.validate = function(data) {
141 |       var hit, i, len, ref, val;
142 |       if (!data) {
143 |         return false;
144 |       } else {
145 |         this.fails = [];
146 | 	if (data.hits && data.hits.hits.length === 0 || data.length === 0) {
147 |             log.debug("AnomaliesValidator.validate: no hits for query! ");
148 |             return false;
149 |         }
150 | 	if (data.hits && data.hits.hits) { data = data.hits.hits; }
151 |         ref = data;
152 | 	var series = {};
153 |         for (i = 0, len = ref.length; i < len; i++) {
154 |           hit = ref[i];
155 |           val = hit[this.fieldName] || hit._source[this.fieldName];
156 | 	  series[i] = val;
157 |         }
158 | 	var res = clustering.nearness(series);
159 | 	log.debug('Analyzing series', series, res.outliers);
160 | 
161 |           if ( Object.getOwnPropertyNames(res.outliers).length === 0 ) {
162 |             this.fails.length = 0;
163 |           } else {
164 | 		// Iterate Outliers
165 | 		for ( var p in res.outliers ) { this.fails.push(series[p]); }
166 |           }
167 |           if (this.fails.length > this.tolerance) {
168 |             log.debug("AnomaliesValidator.validate: anomalies found in series!");
169 |             return false;
170 |           }
171 | 	return;
172 |       }
173 |       return true;
174 |     };
175 | 
176 | 
177 |     /**
178 |      * Return human readable error message describing alarm reason. Empty if no
179 |      * validation failed yet.
180 |      *
181 |      * @method getMessage
182 |      * @return {String}
183 |      */
184 | 
185 |     AnomaliesValidator.prototype.getMessage = function() {
186 |       return "'" + this.fieldName + "' series contains '" + this.fails.length + "' anomalies! [" + this.fails + "]";
187 |     };
188 | 
189 |     return AnomaliesValidator;
190 | 
191 |   })(Validator);
192 | 
193 | }).call(this);
194 | 


--------------------------------------------------------------------------------
/src/validators/dropeak.js:
--------------------------------------------------------------------------------
  1 | (function() {
  2 |   var DropeakValidator, Validator, log;
  3 |   var bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; };
  4 |   var extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
  5 |   var hasProp = {}.hasOwnProperty;
  6 | 
  7 |   log = require("loglevel");
  8 | 
  9 |   Validator = require("../validator");
 10 | 
 11 |   /**
 12 |    * The DropeakValidator takes an elasticsearch query result and compares it to a
 13 |    * defined expectation (default is "value is within range of min/max for n
 14 |    * times").
 15 |    *
 16 |    * @class   DropeakValidator
 17 |    * @extends  Validator
 18 |    */
 19 | 
 20 |   module.exports = DropeakValidator = (function(superClass) {
 21 |     extend(DropeakValidator, superClass);
 22 | 
 23 |     /**
 24 |      * Create a new DropeakValidator with the given options.
 25 |      * @constructor
 26 |      * @param  fieldName {String}  name of the result field (key) to use as comaprison value
 27 |      * @param  drop      {int}     minimum allowed value (= lower bound)
 28 |      * @param  peak      {int}     maximum allowed value (= upper bound)
 29 |      * @param  tolerance {int}     maximum allowed number of consecutive values that do not match the expectation
 30 |      */
 31 | 
 32 |     function DropeakValidator(config) {
 33 |       this.config = config != null ? config : {};
 34 |       this.fieldName = this.config.fieldName;
 35 |       this.drop = this.config.drop;
 36 |       this.peak = this.config.peak;
 37 |       this.tolerance = this.config.tolerance;
 38 |       this.validate = bind(this.validate, this);
 39 |       this.fails = [];
 40 |       if (!this.fieldName || this.drop === null || this.peak === null || this.tolerance === null) {
 41 |         throw new Error("invalid number of options");
 42 |       }
 43 |     }
 44 | 
 45 |     /*
 46 |      * Validate the given elasticsearch query result against the expectation.
 47 |      *
 48 |      * @method validate
 49 |      * @param  data  {Object}  elasticsearch query result
 50 |      */
 51 | 
 52 |     DropeakValidator.prototype.validate = function(data) {
 53 |       var hit, i, len, ref, val;
 54 |       if (!data) {
 55 |         return false;
 56 |       } else {
 57 |         this.fails = [];
 58 | 	if (data.hits && data.hits.hits.length == 0||!data) {
 59 |             log.debug("DropeakValidator.validate: no hits for query! ");
 60 |             return false;
 61 |         }
 62 | 	if (data.hits && data.hits.hits) { data = data.hits.hits; }
 63 |         ref = data;
 64 |         for (i = 1, len = ref.length-1; i < len; i++) {
 65 |           val = ref[i]._source[this.fieldName];
 66 |           pre = ref[i-1]._source[this.fieldName];
 67 |           pos = ref[i+1]._source[this.fieldName];
 68 | 	  log.debug("DropeakValidator.validate: checking: " + val );
 69 | 
 70 | 	    if ((val * this.drop) < pre && (val * this.drop) > pos ) {
 71 | 	            log.debug("DropeakValidator.validate: peak detected! "+val);
 72 | 	            this.fails.push(val);
 73 | 	    } else if ((val / this.peak) > pre && (val / this.peak) > pos ) {
 74 |                     log.debug("DropeakValidator.validate: drop detected! "+val);
 75 |                     this.fails.push(val);
 76 |             } 
 77 | 
 78 |           if (this.fails.length > this.tolerance) {
 79 |             log.debug("DropeakValidator.validate: more than " + this.tolerance + " consecutive peaks/drops detected!");
 80 |             return false;
 81 |           }
 82 |         }
 83 |       }
 84 |       return true;
 85 |     };
 86 | 
 87 | 
 88 |     /**
 89 |      * Return human readable error message describing alarm reason. Empty if no
 90 |      * validation failed yet.
 91 |      *
 92 |      * @method getMessage
 93 |      * @return {String}
 94 |      */
 95 | 
 96 |     DropeakValidator.prototype.getMessage = function() {
 97 |       return "'" + this.fieldName + "' peaks/drops detected '" + (this.fails.length) + "' consecutive times: '" + (this.fails.join(',')) + "'";
 98 |     };
 99 | 
100 |     return DropeakValidator;
101 | 
102 |   })(Validator);
103 | 
104 | }).call(this);
105 | 


--------------------------------------------------------------------------------
/src/validators/eval.js:
--------------------------------------------------------------------------------
  1 | (function() {
  2 |   var EvalValidator, Validator, log, jexl;
  3 |   var bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; };
  4 |   var extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
  5 |   var hasProp = {}.hasOwnProperty;
  6 | 
  7 |   log = require("loglevel");
  8 | 
  9 |   Jexl = require("jexl");
 10 |   var jexl = new Jexl.Jexl();
 11 | 
 12 |   Validator = require("../validator");
 13 | 
 14 |   /**
 15 |    * The EvalValidator takes a query result and compares it to a
 16 |    * defined expectation using JEXL expressions.
 17 |    *
 18 |    * @class   EvalValidator
 19 |    * @extends  Validator
 20 |    */
 21 | 
 22 |   module.exports = EvalValidator = (function(superClass) {
 23 |     extend(EvalValidator, superClass);
 24 | 
 25 |     /**
 26 |      * Create a new EvalValidator with the given options.
 27 |      * @constructor
 28 |      * @param  fieldName {String}  name of the result field (key) to use as comaprison value
 29 |      * @param  exp       {String}  JEXL expression.
 30 |      * @param  tolerance {int}     maximum allowed number of consecutive values that do not match the expectation
 31 |      */
 32 |     // function EvalValidator(fieldName, exp, tolerance) {
 33 |     function EvalValidator(config) {
 34 |       this.config = config != null ? config : {};
 35 |       this.fieldName = this.config.fieldName;
 36 |       this.exp = this.config.exp;
 37 |       this.tolerance = this.config.tolerance || 0;
 38 |       this.validate = bind(this.validate, this);
 39 |       this.fails = [];
 40 |       if (!this.fieldName || !this.exp || this.tolerance === null) {
 41 |         throw new Error("invalid number of options");
 42 |       }
 43 |     }
 44 | 
 45 | 
 46 |     /*
 47 |      * Validate the given query result against the expectation.
 48 |      *
 49 |      * @method validate
 50 |      * @param  data  {Object}  query result
 51 |      */
 52 | 
 53 |     EvalValidator.prototype.validate = function(data) {
 54 |       var hit, i, len, ref, val;
 55 |       if (!data) {
 56 |         return false;
 57 |       } else {
 58 | 	if (data.hits && data.hits.hits.length == 0||!data) {
 59 |             log.debug("EvalValidator.validate: no hits for query! ");
 60 |             return false;
 61 |         }
 62 | 
 63 | 	if (data.hits && data.hits.hits) {
 64 | 		this.resp = { data: data.hits.hits };
 65 | 	} else {
 66 | 		this.resp = { data: data };
 67 |         }
 68 | 
 69 | 	jexl.eval(this.exp, this.resp).then(function(res) {
 70 | 		if (!res) { 
 71 | 		        throw new Error("No Data");
 72 | 		}
 73 | 		log.debug("EvalValidator.validate result: "+res);
 74 | 		if (this.fails) { 
 75 | 			this.fails.push(res);
 76 | 			if (this.fails.length > this.tolerance) {
 77 | 	        	    log.debug("EvalValidator.validate: more than " + this.tolerance + " results occured");
 78 | 	        	    return false;
 79 | 	        	}
 80 | 		}
 81 | 	}.bind(this));
 82 | 	return;
 83 |       }
 84 |       return true;
 85 |     };
 86 | 
 87 | 
 88 |     /**
 89 |      * Return human readable error message describing alarm reason. Empty if no
 90 |      * validation failed yet.
 91 |      *
 92 |      * @method getMessage
 93 |      * @return {String}
 94 |      */
 95 | 
 96 |     EvalValidator.prototype.getMessage = function() {
 97 |       return "'" + this.fieldName + "' JAXL Expression '" + this.exp + "' for '" + (this.fails.length) + "' consecutive times: '" + (this.fails.join(',')) + "'";
 98 |     };
 99 | 
100 |     return EvalValidator;
101 | 
102 |   })(Validator);
103 | 
104 | }).call(this);
105 | 


--------------------------------------------------------------------------------
/src/validators/range.js:
--------------------------------------------------------------------------------
  1 | (function() {
  2 |   var RangeValidator, Validator, log;
  3 |   var bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; };
  4 |   var extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
  5 |   var hasProp = {}.hasOwnProperty;
  6 | 
  7 |   log = require("loglevel");
  8 | 
  9 |   Validator = require("../validator");
 10 | 
 11 |   /**
 12 |    * The RangeValidator takes an elasticsearch query result and compares it to a
 13 |    * defined expectation (default is "value is within range of min/max for n
 14 |    * times").
 15 |    *
 16 |    * @class   RangeValidator
 17 |    * @extends  Validator
 18 |    */
 19 | 
 20 |   module.exports = RangeValidator = (function(superClass) {
 21 |     extend(RangeValidator, superClass);
 22 | 
 23 |     /**
 24 |      * Create a new RangeValidator with the given options.
 25 |      * @constructor
 26 |      * @param  fieldName {String}  name of the result field (key) to use as comaprison value
 27 |      * @param  min       {int}     minimum allowed value (= lower bound)
 28 |      * @param  max       {int}     maximum allowed value (= upper bound)
 29 |      * @param  tolerance {int}     maximum allowed number of consecutive values that do not match the expectation
 30 |      */
 31 |     // function RangeValidator(fieldName, min, max, tolerance) {
 32 |     function RangeValidator(config) {
 33 |       this.config = config != null ? config : {};
 34 |       this.fieldName = this.config.fieldName;
 35 |       this.min = this.config.min;
 36 |       this.max = this.config.max;
 37 |       this.tolerance = this.config.tolerance;
 38 |       this.validate = bind(this.validate, this);
 39 |       this.fails = [];
 40 |       if (!this.fieldName || this.min === null || this.max === null || this.tolerance === null) {
 41 |         throw new Error("invalid number of options");
 42 |       }
 43 |     }
 44 | 
 45 | 
 46 |     /*
 47 |      * Validate the given elasticsearch query result against the expectation.
 48 |      *
 49 |      * @method validate
 50 |      * @param  data  {Object}  elasticsearch query result
 51 |      */
 52 | 
 53 |     RangeValidator.prototype.validate = function(data) {
 54 |       var hit, i, len, ref, val;
 55 |       if (!data) {
 56 |         return false;
 57 |       } else {
 58 |         this.fails = [];
 59 | 	if (data.hits && data.hits.hits.length == 0||data.rowsLength == 0) {
 60 |             log.debug("RangeValidator.validate: no hits for query! ");
 61 |             return false;
 62 |         }
 63 | 	if (data.hits && data.hits.hits) {
 64 | 		data = data.hits.hits;
 65 | 	}
 66 |         ref = data;
 67 |         for (i = 0, len = ref.length; i < len; i++) {
 68 |           hit = ref[i];
 69 |           val = hit[this.fieldName] || hit._source[this.fieldName];
 70 |           log.debug("RangeValidator.validate: val " + val);
 71 |           if ((this.max && val > this.max) || (this.min && val < this.min)) {
 72 |             log.debug("RangeValidator.validate: "+val+" exceeds range");
 73 |             this.fails.push(val);
 74 | 	  }
 75 |           if (this.fails.length > this.tolerance) {
 76 |             log.debug("RangeValidator.validate: more than " + this.tolerance + " consecutive fails occured");
 77 |             return false;
 78 |           }
 79 |         }
 80 | 	return;
 81 |       }
 82 |       return true;
 83 |     };
 84 | 
 85 | 
 86 |     /**
 87 |      * Return human readable error message describing alarm reason. Empty if no
 88 |      * validation failed yet.
 89 |      *
 90 |      * @method getMessage
 91 |      * @return {String}
 92 |      */
 93 | 
 94 |     RangeValidator.prototype.getMessage = function() {
 95 |       return "'" + this.fieldName + "' outside range '" + this.min + "-" + this.max + "' for '" + (this.fails.length) + "' consecutive times: '" + (this.fails.join(',')) + "'";
 96 |     };
 97 | 
 98 |     return RangeValidator;
 99 | 
100 |   })(Validator);
101 | 
102 | }).call(this);
103 | 


--------------------------------------------------------------------------------
/src/workers/cassandra.js:
--------------------------------------------------------------------------------
  1 | (function() {
  2 |   var Worker, events, http, log,
  3 |     bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; },
  4 |     extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
  5 |     hasProp = {}.hasOwnProperty;
  6 | 
  7 |   log = require("loglevel");
  8 | 
  9 |   http = require("http");
 10 |   url = require("url");
 11 |   events = require("events");
 12 | 
 13 |   var cassandra = require('cassandra-driver');
 14 | 
 15 | 
 16 |   /**
 17 |    * The Worker does most of the magic. It connects to cassandra, queries
 18 |    * data, analyzes the result, compares it to the expectation and raises an alarm
 19 |    * when appropriate.
 20 |    *
 21 |    * @class    Worker
 22 |    * @extends  events.EventEmitter
 23 |    */
 24 | 
 25 |   module.exports = Worker = (function(superClass) {
 26 |     extend(Worker, superClass);
 27 | 
 28 |     Worker.ResultCodes = {
 29 |       Success: {
 30 |         code: 0,
 31 |         label: "SUCCESS"
 32 |       },
 33 |       ValidationFailed: {
 34 |         code: 1,
 35 |         label: "ALARM_VALIDATION_FAILED"
 36 |       },
 37 |       NoResults: {
 38 |         code: 2,
 39 |         label: "ALARM_NO_RESULTS_RECEIVED"
 40 |       },
 41 |       NotFound: {
 42 |         code: 4,
 43 |         label: "ALARM_NOT_FOUND_404"
 44 |       },
 45 |       InvalidResponse: {
 46 |         code: 5,
 47 |         label: "ALARM_INVALID_RESPONSE"
 48 |       },
 49 |       ConnectionRefused: {
 50 |         code: 6,
 51 |         label: "ALARM_CONNECTION_REFUSED"
 52 |       },
 53 |       UnhandledError: {
 54 |         code: 99,
 55 |         label: "ALARM_UNHANDLED_ERROR"
 56 |       }
 57 |     };
 58 | 
 59 | 
 60 |     /**
 61 |      * Create a new Worker, prepare data, setup request options.
 62 |      *
 63 |      * @constructor
 64 |      * @param  id        {String}  identifies this individual Worker instance
 65 |      * @param  host      {String}  cassandra hostname to connect to
 66 |      * @param  cqlquery  {Object}  valid cassandra query
 67 |      * @param  params    {Object}  valid variables for query
 68 |      * @param  validator {ResultValidator} a validator object that takes the response and compares it against a given expectation
 69 |      */
 70 | 
 71 |     function Worker(id, host, cqlquery, params, validator) {
 72 | 
 73 |       this.id = id;
 74 |       this.host = host;
 75 |       this.cqlquery = cqlquery;
 76 |       this.params = params;
 77 |       this.validator = validator;
 78 |       this.onError = bind(this.onError, this);
 79 |       this.onResponse = bind(this.onResponse, this);
 80 |       this.raiseAlarm = bind(this.raiseAlarm, this);
 81 |       this.start = bind(this.start, this);
 82 |       if (!this.id || !this.host || !this.cqlquery || !this.validator) {
 83 | 	console.log(id,host,cqlquery,params );
 84 |         throw new Error("Worker.constructor: invalid number of required options received: " + (JSON.stringify(arguments)));
 85 |       }
 86 |     }
 87 | 
 88 | 
 89 |     /**
 90 |      * Execute request and hand over control to onResponse callback.
 91 |      *
 92 |      * @method start
 93 |      */
 94 | 
 95 |     Worker.prototype.start = function() {
 96 |       var data, e, error1;
 97 |       this.options = {
 98 |         contactPoints: [ this.host||"127.0.0.1" ],
 99 |       };
100 |       log.debug("Worker(" + this.id + ").sendCQLRequest: connecting to cassandra at: " + this.host);
101 |       try {
102 | 
103 | 	var client = new cassandra.Client(this.options);
104 | 	client.connect()
105 |   	.then(function () {
106 | 	        log.debug("Worker(" + this.id + ").sendCQLRequest: cqlquery is: ", this.cqlquery, this.params);
107 | 	  	return client.execute(this.cqlquery);
108 |   	}.bind(this))
109 | 	.then(function(data){
110 | 		this.onResponse(data);
111 | 		return client.shutdown();
112 | 	}.bind(this))
113 | 	.catch(function(err){
114 | 	        return log.error("Worker(" + this.id + ").start: unhandled error: " + err);
115 | 		return client.shutdown();
116 | 	}.bind(this));
117 | 	return true;
118 | 
119 |       } catch(err){
120 | 	return log.error("Worker(" + this.id + ").start: unhandled error: " + err);
121 |       }
122 | 
123 |     };
124 | 
125 | 
126 |     /**
127 |      * Gets passed CQL response data (as object) and pre-validates the contents.
128 |      * If data is invalid or result is empty an error is raised. Valid results
129 |      * are handed over to the ResultValidator for further analysis. If any alarm
130 |      * condition is met, raiseAlarm is called with the appropriate alarm.
131 |      *
132 |      * @method handleResponseData
133 |      * @param  data  {Object}  result set as returned by ES
134 |      */
135 | 
136 |     Worker.prototype.handleResponseData = function(data) {
137 |       var numHits, rc, result;
138 |       result = null;
139 |       var alarms = [];
140 |       rc = Worker.ResultCodes;
141 |       if (!data || typeof data === "undefined") {
142 |         result = rc.InvalidResponse;
143 |       } else {
144 |         log.debug("Worker(" + this.id + ").onResponse: cql query returned " + JSON.stringify(data) );
145 | 	data = JSON.parse(JSON.stringify(data));
146 |         if (data === '') {
147 |           result = rc.NoResults;
148 | 	  return false;
149 |         } else {
150 | 		// Iterate Validators
151 | 		this.validator.forEach(function(validator, i){
152 | 			if (!validator.validate(data)) {
153 | 		          	result = rc.ValidationFailed;
154 | 				alarms.push(i);
155 | 			        // return false;
156 | 		        } else {
157 | 		          result = rc.Success;
158 | 		        }
159 | 		});
160 | 
161 | 	      // if (result === rc.Success) {
162 | 	      if (alarms.length === 0) {
163 | 	        return true;
164 | 	      } else {
165 | 
166 | 		// Iterate Alarms
167 | 		for (idx = 0; idx < alarms.length; ++idx) {
168 | 		        this.raiseAlarm(result.label + ": " + (this.validator[idx].getMessage()));
169 | 		}
170 | 	        process.exitCode = result.code;
171 | 	           return false;
172 | 	      }
173 | 	}
174 |       }
175 | 
176 |     };
177 | 
178 | 
179 |     /**
180 |      * Raise alarm - emits "alarm" event that can be handled by interested
181 |      * listeners.
182 |      *
183 |      * @method raiseAlarm
184 |      * @emits  alarm
185 |      * @param  message {String}  error message
186 |      * @param  data    {object}  any additional data
187 |      */
188 | 
189 |     Worker.prototype.raiseAlarm = function(message) {
190 |       log.debug("Worker(" + this.id + ").raiseAlarm: raising alarm: " + message);
191 |       return this.emit("alarm", message, {
192 |         name: this.id
193 |       });
194 |     };
195 | 
196 | 
197 |     /**
198 |      * http.request: success callback
199 |      *
200 |      * @method onResponse
201 |      */
202 | 
203 |     Worker.prototype.onResponse = function(body) {
204 |        // log.debug("Worker(" + this.id + ").onResponse: response was: ", body);
205 |          try {
206 |               return this.handleResponseData(body.rows);
207 | 	      return;
208 |          } catch (error1) {
209 |               e = error1;
210 |               log.error("Worker(" + this.id + ").onResponse: failed to parse response data",e);
211 | 	      this.raiseAlarm("" + Worker.ResultCodes.NotFound.label);
212 | 	      process.exitCode = Worker.ResultCodes.NotFound.code;
213 | 	      return process.exit();
214 |          }
215 |     };
216 | 
217 | 
218 |     /**
219 |      * http.request: error callback
220 |      *
221 |      * @method onError
222 |      */
223 | 
224 |     Worker.prototype.onError = function(error) {
225 |       if (error.code === "ECONNREFUSED") {
226 |         log.error("ERROR: connection refused, please make sure Cassandra is running and accessible under " + this.options.host);
227 |         this.raiseAlarm("" + Worker.ResultCodes.ConnectionRefused.label);
228 |         process.exitCode = Worker.ResultCodes.ConnectionRefused.code;
229 |       } else {
230 |         log.debug("Worker(" + this.id + ").onError: unhandled error: ", error);
231 |         this.raiseAlarm(Worker.ResultCodes.UnhandledError.label + ": " + error);
232 |         process.exitCode = Worker.ResultCodes.UnhandledError.code;
233 |       }
234 |       return;
235 |     };
236 | 
237 |     return Worker;
238 | 
239 |   })(events.EventEmitter);
240 | 
241 | }).call(this);
242 | 


--------------------------------------------------------------------------------
/src/workers/elastic.js:
--------------------------------------------------------------------------------
  1 | (function() {
  2 |   var Worker, events, http, log,
  3 |     bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; },
  4 |     extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
  5 |     hasProp = {}.hasOwnProperty;
  6 | 
  7 |   log = require("loglevel");
  8 | 
  9 |   http = require("http");
 10 |   url = require("url");
 11 |   events = require("events");
 12 | 
 13 | 
 14 |   /**
 15 |    * The Worker does most of the magic. It connects to elasticsearch, queries
 16 |    * data, analyzes the result, compares it to the expectation and raises an alarm
 17 |    * when appropriate.
 18 |    *
 19 |    * @class    Worker
 20 |    * @extends  events.EventEmitter
 21 |    */
 22 | 
 23 |   module.exports = Worker = (function(superClass) {
 24 |     extend(Worker, superClass);
 25 | 
 26 |     Worker.ResultCodes = {
 27 |       Success: {
 28 |         code: 0,
 29 |         label: "SUCCESS"
 30 |       },
 31 |       ValidationFailed: {
 32 |         code: 1,
 33 |         label: "ALARM_VALIDATION_FAILED"
 34 |       },
 35 |       NoResults: {
 36 |         code: 2,
 37 |         label: "ALARM_NO_RESULTS_RECEIVED"
 38 |       },
 39 |       NotFound: {
 40 |         code: 4,
 41 |         label: "ALARM_NOT_FOUND_404"
 42 |       },
 43 |       InvalidResponse: {
 44 |         code: 5,
 45 |         label: "ALARM_INVALID_RESPONSE"
 46 |       },
 47 |       ConnectionRefused: {
 48 |         code: 6,
 49 |         label: "ALARM_CONNECTION_REFUSED"
 50 |       },
 51 |       UnhandledError: {
 52 |         code: 99,
 53 |         label: "ALARM_UNHANDLED_ERROR"
 54 |       }
 55 |     };
 56 | 
 57 | 
 58 |     /**
 59 |      * Create a new Worker, prepare data, setup request options.
 60 |      *
 61 |      * @constructor
 62 |      * @param  id        {String}  identifies this individual Worker instance
 63 |      * @param  host      {String}  elasticsearch hostname to connect to
 64 |      * @param  port      {String}  elasticsearch port to connect to
 65 |      * @param  path      {String}  elasticsearch path (in form /{index}/{type})
 66 |      * @param  query     {Object}  valid elasticsearch query
 67 |      * @param  aggs      {Object}  valid elasticsearch aggs
 68 |      * @param  validator {ResultValidator} a validator object that takes the response and compares it against a given expectation
 69 |      */
 70 | 
 71 |     function Worker(id, host, port, path, query, aggs, validator) {
 72 | 
 73 |       if( url.parse(host).auth  ) {
 74 | 		this.auth = url.parse(host).auth;
 75 | 		this.host = url.parse(host).host;
 76 |       } else {
 77 | 	        this.host = host;
 78 |       }
 79 | 
 80 |       this.id = id;
 81 |       this.port = port;
 82 |       this.path = path;
 83 |       this.query = query;
 84 |       this.aggs = aggs;
 85 |       this.validator = validator;
 86 |       this.onError = bind(this.onError, this);
 87 |       this.onResponse = bind(this.onResponse, this);
 88 |       this.raiseAlarm = bind(this.raiseAlarm, this);
 89 |       this.start = bind(this.start, this);
 90 |       if (!this.id || !this.host || !this.port || !this.path || !this.query || !this.aggs || !this.validator) {
 91 |         throw new Error("Worker.constructor: invalid number of options received: " + (JSON.stringify(arguments)));
 92 |       }
 93 |     }
 94 | 
 95 | 
 96 |     /**
 97 |      * Execute request and hand over control to onResponse callback.
 98 |      *
 99 |      * @method start
100 |      */
101 | 
102 |     Worker.prototype.start = function() {
103 |       var data, e, error1;
104 |       data = {
105 |         query: this.query,
106 |       };
107 |       if (Object.keys(this.aggs).length) data.aggs = this.aggs;
108 |       data = JSON.stringify(data);
109 |       this.options = {
110 |         host: this.host,
111 |         port: this.port,
112 |         path: this.path + "/_search",
113 |         method: "POST",
114 | 	auth: this.auth,
115 |         headers: {
116 |           "Content-Type": "application/json",
117 |           "Content-Length": Buffer.byteLength(data)
118 |         }
119 |       };
120 |       log.debug("Worker(" + this.id + ").sendESRequest: connecting to elasticsearch at: " + this.host + ":" + this.port + this.path);
121 |       try {
122 |         this.request = http.request(this.options, this.onResponse);
123 |         this.request.on("error", this.onError);
124 |         log.debug("Worker(" + this.id + ").sendESRequest: query data is: ", data);
125 |         this.request.write(data);
126 |         this.request.end();
127 |         return true;
128 |       } catch (error1) {
129 |         e = error1;
130 |         return log.error("Worker(" + this.id + ").start: unhandled error: " + e.message);
131 |       }
132 |     };
133 | 
134 | 
135 |     /**
136 |      * Gets passed ES response data (as object) and pre-validates the contents.
137 |      * If data is invalid or result is empty an error is raised. Valid results
138 |      * are handed over to the ResultValidator for further analysis. If any alarm
139 |      * condition is met, raiseAlarm is called with the appropriate alarm.
140 |      *
141 |      * @method handleResponseData
142 |      * @param  data  {Object}  result set as returned by ES
143 |      */
144 | 
145 |     Worker.prototype.handleResponseData = function(data) {
146 |       var numHits, rc, result;
147 |       result = null;
148 |       var alarms = [];
149 |       rc = Worker.ResultCodes;
150 |       if (!data || typeof data.hits === "undefined") {
151 |         result = rc.InvalidResponse;
152 |       } else {
153 |         numHits = data.hits.total;
154 |         log.debug("Worker(" + this.id + ").onResponse: query returned " + numHits + " hits");
155 | 
156 | 
157 |         if (numHits === 0) {
158 |           result = rc.NoResults;
159 | 	  return false;
160 |         } else {
161 | 		// Iterate Validators
162 | 		this.validator.forEach(function(validator, i){
163 | 			if (!validator.validate(data)) {
164 | 		          	result = rc.ValidationFailed;
165 | 				alarms.push(i);
166 | 			        // return false;
167 | 		        } else {
168 | 		          result = rc.Success;
169 | 		        }
170 | 		});
171 | 
172 | 	      // if (result === rc.Success) {
173 | 	      if (alarms.length == 0) {
174 | 	        return true;
175 | 	      } else {
176 | 
177 | 		// Iterate Alarms
178 | 		for (idx = 0; idx < alarms.length; ++idx) {
179 | 		        this.raiseAlarm(result.label + ": " + (this.validator[idx].getMessage()));
180 | 		}
181 | 	        process.exitCode = result.code;
182 | 	           return false;
183 | 	      }
184 | 	}
185 |       }
186 | 
187 |     };
188 | 
189 | 
190 |     /**
191 |      * Raise alarm - emits "alarm" event that can be handled by interested
192 |      * listeners.
193 |      *
194 |      * @method raiseAlarm
195 |      * @emits  alarm
196 |      * @param  message {String}  error message
197 |      * @param  data    {object}  any additional data
198 |      */
199 | 
200 |     Worker.prototype.raiseAlarm = function(message) {
201 |       log.debug("Worker(" + this.id + ").raiseAlarm: raising alarm: " + message);
202 |       return this.emit("alarm", message, {
203 |         name: this.id
204 |       });
205 |     };
206 | 
207 | 
208 |     /**
209 |      * http.request: success callback
210 |      *
211 |      * @method onResponse
212 |      */
213 | 
214 |     Worker.prototype.onResponse = function(response) {
215 |       var body;
216 |       log.debug("Worker(" + this.id + ").onResponse: status is " + response.statusCode);
217 |       if (response.statusCode === 200) {
218 |         body = "";
219 |         response.setEncoding("utf8");
220 |         response.on("data", (function(_this) {
221 |           return function(chunk) {
222 |             return body += chunk;
223 |           };
224 |         })(this));
225 |         return response.on("end", (function(_this) {
226 |           return function(error) {
227 |             var data, e, error1;
228 |             log.debug("Worker(" + _this.id + ").onResponse: response was: ", body);
229 |             try {
230 |               data = JSON.parse(body);
231 |             } catch (error1) {
232 |               e = error1;
233 |               log.error("Worker(" + _this.id + ").onResponse: failed to parse response data");
234 |             }
235 |             if (data) {
236 |               return _this.handleResponseData(data);
237 |             }
238 |           };
239 |         })(this));
240 |       } else {
241 |         this.raiseAlarm("" + Worker.ResultCodes.NotFound.label);
242 |         process.exitCode = Worker.ResultCodes.NotFound.code;
243 |         this.request.end();
244 |         return process.exit();
245 |       }
246 |     };
247 | 
248 | 
249 |     /**
250 |      * http.request: error callback
251 |      *
252 |      * @method onError
253 |      */
254 | 
255 |     Worker.prototype.onError = function(error) {
256 |       if (error.code === "ECONNREFUSED") {
257 |         log.error("ERROR: connection refused, please make sure elasticsearch is running and accessible under " + this.options.host + ":" + this.options.port);
258 |         this.raiseAlarm("" + Worker.ResultCodes.ConnectionRefused.label);
259 |         process.exitCode = Worker.ResultCodes.ConnectionRefused.code;
260 |       } else {
261 |         log.debug("Worker(" + this.id + ").onError: unhandled error: ", error);
262 |         this.raiseAlarm(Worker.ResultCodes.UnhandledError.label + ": " + error);
263 |         process.exitCode = Worker.ResultCodes.UnhandledError.code;
264 |       }
265 |       return this.request.end();
266 |     };
267 | 
268 |     return Worker;
269 | 
270 |   })(events.EventEmitter);
271 | 
272 | }).call(this);
273 | 


--------------------------------------------------------------------------------
/test/app.js:
--------------------------------------------------------------------------------
  1 | // Generated by CoffeeScript 1.10.0
  2 | (function() {
  3 |   var App, ConsoleReporter, Validator, Worker, app, assert, configMock, loglevelMock, mockery, ref;
  4 | 
  5 |   mockery = require("mockery");
  6 | 
  7 |   assert = require("chai").assert;
  8 | 
  9 |   ref = [], app = ref[0], configMock = ref[1];
 10 | 
 11 |   loglevelMock = {
 12 |     debug: function(str) {
 13 |       return this.strDebug = str;
 14 |     },
 15 |     error: function(str) {
 16 |       console.error(str);
 17 |       return this.strError = str;
 18 |     }
 19 |   };
 20 | 
 21 |   configMock = {
 22 |     name: "Test",
 23 |     elasticsearch: {
 24 |       host: "localhost",
 25 |       port: 9200,
 26 |       index: "_all",
 27 |       type: "type"
 28 |     },
 29 |     query: {},
 30 |     reporters: {
 31 |       console: {}
 32 |     },
 33 |     validator: {
 34 |       fieldName: "prop",
 35 |       min: 10,
 36 |       max: 30,
 37 |       tolerance: 5
 38 |     }
 39 |   };
 40 | 
 41 |   mockery.registerMock("loglevel", loglevelMock);
 42 | 
 43 |   mockery.registerAllowables(["../reporter", "../src/reporter", "./reporters/console", "../src/reporters/console", "../worker", "../src/worker"]);
 44 | 
 45 |   mockery.enable({
 46 |     useCleanCache: true
 47 |   });
 48 | 
 49 |   App = require("../src/app");
 50 | 
 51 |   ConsoleReporter = require("../src/reporters/console");
 52 | 
 53 |   Worker = require("../src/worker");
 54 | 
 55 |   Validator = require("../src/validator");
 56 | 
 57 |   describe("App", function() {
 58 |     describe("init", function() {
 59 |       var stub;
 60 |       stub = [][0];
 61 |       beforeEach(function() {
 62 |         return stub = {
 63 |           name: 1,
 64 |           elasticsearch: 1,
 65 |           query: 1,
 66 |           reporters: 1,
 67 |           validator: 1
 68 |         };
 69 |       });
 70 |       it("should throw an error if config.name is missing", function() {
 71 |         var init;
 72 |         delete stub.name;
 73 |         init = function() {
 74 |           return new App(stub);
 75 |         };
 76 |         return assert["throw"](init, Error, "config.name missing");
 77 |       });
 78 |       it("should throw an error if config.elasticsearch is missing", function() {
 79 |         var init;
 80 |         delete stub.elasticsearch;
 81 |         init = function() {
 82 |           return new App(stub);
 83 |         };
 84 |         return assert["throw"](init, Error, "config.elasticsearch missing");
 85 |       });
 86 |       it("should throw an error if config.query is missing", function() {
 87 |         var init;
 88 |         delete stub.query;
 89 |         init = function() {
 90 |           return new App(stub);
 91 |         };
 92 |         return assert["throw"](init, Error, "config.query missing");
 93 |       });
 94 |       it("should throw an error if config.reporters is missing", function() {
 95 |         var init;
 96 |         delete stub.reporters;
 97 |         init = function() {
 98 |           return new App(stub);
 99 |         };
100 |         return assert["throw"](init, Error, "config.reporters missing");
101 |       });
102 |       return it("should throw an error if config.validator is missing", function() {
103 |         var init;
104 |         delete stub.validator;
105 |         init = function() {
106 |           return new App(stub);
107 |         };
108 |         return assert["throw"](init, Error, "config.validator missing");
109 |       });
110 |     });
111 |     describe("App.createWorker", function() {
112 |       it("should create a Worker from a given config", function() {
113 |         return assert.instanceOf(App.createWorker("testworker", configMock.elasticsearch, configMock.query, {}), Worker);
114 |       });
115 |       it("should return null if any of the options are missing", function() {
116 |         assert.isNull(App.createWorker(null, {}, {}, {}));
117 |         assert.isNull(App.createWorker("name", null, {}, {}));
118 |         assert.isNull(App.createWorker("name", {}, null, {}));
119 |         return assert.isNull(App.createWorker("name", {}, {}, null));
120 |       });
121 |       return it("should return null if the Worker can't be created", function() {
122 |         return assert.isNull(App.createWorker());
123 |       });
124 |     });
125 |     describe("App.createValidator", function() {
126 |       it("should create a Validator from a given config", function() {
127 |         return assert.instanceOf(App.createValidator("validator", configMock.validator), Validator);
128 |       });
129 |       return it("should return null if the validator can't be created", function() {
130 |         return assert.isNull(App.createValidator("!_random_garbage_!", {}));
131 |       });
132 |     });
133 |     return describe("App.createReporter", function() {
134 |       it("should create the correct Reporter (ConsoleReporter) from a given config", function() {
135 |         return assert.instanceOf(App.createReporter("console", {}), ConsoleReporter);
136 |       });
137 |       return it("should return null if the reporter can't be created", function() {
138 |         return assert.isNull(App.createReporter("!_random_garbage_!", {}));
139 |       });
140 |     });
141 |   });
142 | 
143 | }).call(this);
144 | 


--------------------------------------------------------------------------------
/test/reporter-console.js:
--------------------------------------------------------------------------------
 1 | // Generated by CoffeeScript 1.10.0
 2 | (function() {
 3 |   var ConsoleReporter, assert, loglevelMock, mockery, reporter;
 4 | 
 5 |   mockery = require("mockery");
 6 | 
 7 |   assert = require("chai").assert;
 8 | 
 9 |   reporter = [][0];
10 | 
11 |   loglevelMock = {
12 |     debug: function(str) {
13 |       return this.strDebug = str;
14 |     },
15 |     error: function(str) {
16 |       return this.strError = str;
17 |     }
18 |   };
19 | 
20 |   mockery.enable({
21 |     useCleanCache: true
22 |   });
23 | 
24 |   mockery.registerMock("loglevel", loglevelMock);
25 | 
26 |   mockery.registerAllowables(["../src/reporters/console", "../reporter"]);
27 | 
28 |   ConsoleReporter = require("../src/reporters/console");
29 | 
30 |   describe("ConsoleReporter", function() {
31 |     beforeEach(function() {
32 |       return reporter = new ConsoleReporter();
33 |     });
34 |     it("should output a log message during construction", function() {
35 |       return assert.include(loglevelMock.strDebug, "creating new instance");
36 |     });
37 |     return it("should log an error when notify is called", function() {
38 |       reporter.notify("mymessage", {
39 |         name: "myname"
40 |       });
41 |       return assert.include(loglevelMock.strError, "'myname' raised alarm: mymessage");
42 |     });
43 |   });
44 | 
45 | }).call(this);
46 | 


--------------------------------------------------------------------------------
/test/reporter-mail.js:
--------------------------------------------------------------------------------
 1 | // Generated by CoffeeScript 1.10.0
 2 | (function() {
 3 |   var MailReporter, assert, childProcessMock, loglevelMock, mockery, reporter;
 4 | 
 5 |   mockery = require("mockery");
 6 | 
 7 |   assert = require("chai").assert;
 8 | 
 9 |   reporter = [][0];
10 | 
11 |   loglevelMock = {
12 |     debug: function(str) {
13 |       return this.strDebug = str;
14 |     },
15 |     error: function(str) {
16 |       return this.strError = str;
17 |     }
18 |   };
19 | 
20 |   childProcessMock = {
21 |     mailCommand: "",
22 |     exec: (function(_this) {
23 |       return function(command, callback) {
24 |         childProcessMock.mailCommand = command;
25 |         return callback(null);
26 |       };
27 |     })(this)
28 |   };
29 | 
30 |   mockery.enable({
31 |     useCleanCache: true
32 |   });
33 | 
34 |   mockery.registerMock("loglevel", loglevelMock);
35 | 
36 |   mockery.registerMock("child_process", childProcessMock);
37 | 
38 |   mockery.registerAllowables(["../src/reporters/console", "../reporter"]);
39 | 
40 |   MailReporter = require("../src/reporters/mail");
41 | 
42 |   describe("MailReporter", function() {
43 |     describe("init", function() {
44 |       it("should output a log message during construction", function() {
45 |         new MailReporter();
46 |         return assert.include(loglevelMock.strDebug, "creating new instance");
47 |       });
48 |       it("should throw an error if no target address is supplied", function() {
49 |         new MailReporter();
50 |         return assert.include(loglevelMock.strError, "requires 'targetAddress'");
51 |       });
52 |       it("should set maxRetries to the supplied value [10]", function() {
53 |         reporter = new MailReporter({
54 |           maxRetries: 10
55 |         });
56 |         return assert.equal(reporter.maxRetries, 10);
57 |       });
58 |       return it("should set maxRetries to 3 if only an e-mail address is defined", function() {
59 |         reporter = new MailReporter({
60 |           targetAddress: "test@example.com"
61 |         });
62 |         return assert.equal(reporter.maxRetries, 3);
63 |       });
64 |     });
65 |     return describe("notify", function() {
66 |       it("should call sendMail with the appropriate message", function() {
67 |         reporter = new MailReporter({
68 |           targetAddress: "test@example.com"
69 |         });
70 |         reporter.notify("myMessage", {
71 |           name: "myname"
72 |         });
73 |         return assert.include(childProcessMock.mailCommand, "myMessage");
74 |       });
75 |       it("should log error if sending mail fails", function() {
76 |         childProcessMock.exec = (function(_this) {
77 |           return function(command, callback) {
78 |             return callback("someError");
79 |           };
80 |         })(this);
81 |         reporter = new MailReporter({
82 |           targetAddress: "test@example.com"
83 |         });
84 |         reporter.notify("myMessage", {
85 |           name: "myname"
86 |         });
87 |         return assert.include(loglevelMock.strError, "mail delivery failed");
88 |       });
89 |       return it("should retry sending the mail on error", function() {});
90 |     });
91 |   });
92 | 
93 | }).call(this);
94 | 


--------------------------------------------------------------------------------
/test/validator.js:
--------------------------------------------------------------------------------
  1 | // Generated by CoffeeScript 1.10.0
  2 | (function() {
  3 |   var Validator, assert, loglevelMock, mockery, worker;
  4 | 
  5 |   mockery = require("mockery");
  6 | 
  7 |   assert = require("chai").assert;
  8 | 
  9 |   worker = [][0];
 10 | 
 11 |   loglevelMock = {
 12 |     debug: function(str) {
 13 |       return this.strDebug = str;
 14 |     },
 15 |     error: function(str) {
 16 |       return this.strError = str;
 17 |     }
 18 |   };
 19 | 
 20 |   mockery.enable({
 21 |     useCleanCache: true
 22 |   });
 23 | 
 24 |   mockery.registerMock("loglevel", loglevelMock);
 25 | 
 26 |   mockery.registerAllowables(["../src/validator"]);
 27 | 
 28 |   Validator = require("../src/validator");
 29 | 
 30 |   describe("Validator", function() {
 31 |     describe("constructor", function() {
 32 |       return it("should break if any argument of [fieldName,min,max,tolerance] is missing", function() {
 33 |         assert["throw"]((function() {
 34 |           return new Validator(null, 10, 20, 5);
 35 |         }), Error, "invalid number of options");
 36 |         assert["throw"]((function() {
 37 |           return new Validator("prop", null, 20, 5);
 38 |         }), Error, "invalid number of options");
 39 |         assert["throw"]((function() {
 40 |           return new Validator("prop", 10, null, 5);
 41 |         }), Error, "invalid number of options");
 42 |         return assert["throw"]((function() {
 43 |           return new Validator("prop", 10, 20, null);
 44 |         }), Error, "invalid number of options");
 45 |       });
 46 |     });
 47 |     return describe("validate", function() {
 48 |       var validator;
 49 |       validator = [][0];
 50 |       beforeEach(function() {
 51 |         return validator = new Validator("prop", 10, 30, 4);
 52 |       });
 53 |       it("should return false if no data is supplied", function() {
 54 |         return assert.isFalse(validator.validate());
 55 |       });
 56 |       it("should return false if 5 consecutive values within the result are below the expectation", function() {
 57 |         var result;
 58 |         result = {
 59 |           hits: {
 60 |             hits: [
 61 |               {
 62 |                 _source: {
 63 |                   prop: 5
 64 |                 }
 65 |               }, {
 66 |                 _source: {
 67 |                   prop: 7
 68 |                 }
 69 |               }, {
 70 |                 _source: {
 71 |                   prop: 6
 72 |                 }
 73 |               }, {
 74 |                 _source: {
 75 |                   prop: 9
 76 |                 }
 77 |               }, {
 78 |                 _source: {
 79 |                   prop: 4
 80 |                 }
 81 |               }
 82 |             ]
 83 |           }
 84 |         };
 85 |         return assert.isFalse(validator.validate(result));
 86 |       });
 87 |       it("should return false if 5 consecutive values within the result are above the expectation", function() {
 88 |         var result;
 89 |         result = {
 90 |           hits: {
 91 |             hits: [
 92 |               {
 93 |                 _source: {
 94 |                   prop: 35
 95 |                 }
 96 |               }, {
 97 |                 _source: {
 98 |                   prop: 37
 99 |                 }
100 |               }, {
101 |                 _source: {
102 |                   prop: 36
103 |                 }
104 |               }, {
105 |                 _source: {
106 |                   prop: 39
107 |                 }
108 |               }, {
109 |                 _source: {
110 |                   prop: 34
111 |                 }
112 |               }
113 |             ]
114 |           }
115 |         };
116 |         return assert.isFalse(validator.validate(result));
117 |       });
118 |       it("should return true if less than 5 consecutive values within the result are below the expectation", function() {
119 |         var result;
120 |         result = {
121 |           hits: {
122 |             hits: [
123 |               {
124 |                 _source: {
125 |                   prop: 5
126 |                 }
127 |               }, {
128 |                 _source: {
129 |                   prop: 7
130 |                 }
131 |               }, {
132 |                 _source: {
133 |                   prop: 6
134 |                 }
135 |               }, {
136 |                 _source: {
137 |                   prop: 9
138 |                 }
139 |               }, {
140 |                 _source: {
141 |                   prop: 11
142 |                 }
143 |               }
144 |             ]
145 |           }
146 |         };
147 |         return assert.isTrue(validator.validate(result));
148 |       });
149 |       it("should return true if less than 5 consecutive values within the result are above the expectation", function() {
150 |         var result;
151 |         result = {
152 |           hits: {
153 |             hits: [
154 |               {
155 |                 _source: {
156 |                   prop: 35
157 |                 }
158 |               }, {
159 |                 _source: {
160 |                   prop: 37
161 |                 }
162 |               }, {
163 |                 _source: {
164 |                   prop: 36
165 |                 }
166 |               }, {
167 |                 _source: {
168 |                   prop: 39
169 |                 }
170 |               }, {
171 |                 _source: {
172 |                   prop: 29
173 |                 }
174 |               }
175 |             ]
176 |           }
177 |         };
178 |         return assert.isTrue(validator.validate(result));
179 |       });
180 |       return it("should return true if all values within the result meet the expectation", function() {
181 |         var result;
182 |         result = {
183 |           hits: {
184 |             hits: [
185 |               {
186 |                 _source: {
187 |                   prop: 12
188 |                 }
189 |               }, {
190 |                 _source: {
191 |                   prop: 17
192 |                 }
193 |               }, {
194 |                 _source: {
195 |                   prop: 22
196 |                 }
197 |               }, {
198 |                 _source: {
199 |                   prop: 23
200 |                 }
201 |               }, {
202 |                 _source: {
203 |                   prop: 27
204 |                 }
205 |               }
206 |             ]
207 |           }
208 |         };
209 |         return assert.isTrue(validator.validate(result));
210 |       });
211 |     });
212 |   });
213 | 
214 | }).call(this);
215 | 


--------------------------------------------------------------------------------
/test/worker.js:
--------------------------------------------------------------------------------
  1 | // Generated by CoffeeScript 1.10.0
  2 | (function() {
  3 |   var Worker, assert, httpMock, httpResponseMock, loglevelMock, mockery, validatorMock, worker;
  4 | 
  5 |   mockery = require("mockery");
  6 | 
  7 |   assert = require("chai").assert;
  8 | 
  9 |   worker = [][0];
 10 | 
 11 |   loglevelMock = {
 12 |     debug: function(str) {
 13 |       return this.strDebug = str;
 14 |     },
 15 |     error: function(str) {
 16 |       return this.strError = str;
 17 |     }
 18 |   };
 19 | 
 20 |   httpResponseMock = {
 21 |     statusCode: 200,
 22 |     responseData: JSON.stringify({
 23 |       foo: "bar"
 24 |     }),
 25 |     on: function(name, callback) {
 26 |       if (name === "data") {
 27 |         return callback(httpResponseMock.responseData);
 28 |       } else if (name === "end") {
 29 |         return callback();
 30 |       }
 31 |     }
 32 |   };
 33 | 
 34 |   httpMock = {
 35 |     request: function(options, callback) {
 36 |       this.requestOptions = options;
 37 |       return {
 38 |         on: function() {},
 39 |         end: function() {
 40 |           return callback(httpResponseMock);
 41 |         },
 42 |         write: (function(_this) {
 43 |           return function(data) {
 44 |             return httpMock.writeData = data;
 45 |           };
 46 |         })(this)
 47 |       };
 48 |     }
 49 |   };
 50 | 
 51 |   validatorMock = {
 52 |     validate: function(data) {
 53 |       return true;
 54 |     },
 55 |     getMessage: function() {
 56 |       return "testmessage";
 57 |     }
 58 |   };
 59 | 
 60 |   mockery.enable({
 61 |     useCleanCache: true
 62 |   });
 63 | 
 64 |   mockery.registerMock("loglevel", loglevelMock);
 65 | 
 66 |   mockery.registerMock("http", httpMock);
 67 | 
 68 |   mockery.registerMock("./validator", validatorMock);
 69 | 
 70 |   mockery.registerAllowables(["../src/worker", "events"]);
 71 | 
 72 |   Worker = require("../src/worker");
 73 | 
 74 |   describe("Worker", function() {
 75 |     describe("constructor", function() {
 76 |       it("should have the assigned id", function() {
 77 |         return assert.equal(new Worker("testworker", "host", 9200, "/_all", {}, validatorMock).id, "testworker", "id property should equal the constructor's first argument");
 78 |       });
 79 |       return it("should break if any argument of [id,host,port,path,query] is missing", function() {
 80 |         assert["throw"]((function() {
 81 |           return new Worker(null, "host", 9200, "/_all", {}, validatorMock);
 82 |         }), Error, "invalid number of options");
 83 |         assert["throw"]((function() {
 84 |           return new Worker("testworker", null, 9200, "/_all", {}, validatorMock);
 85 |         }), Error, "invalid number of options");
 86 |         assert["throw"]((function() {
 87 |           return new Worker("testworker", "host", null, "/_all", {}, validatorMock);
 88 |         }), Error, "invalid number of options");
 89 |         assert["throw"]((function() {
 90 |           return new Worker("testworker", "host", 9200, null, {}, validatorMock);
 91 |         }), Error, "invalid number of options");
 92 |         assert["throw"]((function() {
 93 |           return new Worker("testworker", "host", 9200, "/_all", null, validatorMock);
 94 |         }), Error, "invalid number of options");
 95 |         return assert["throw"]((function() {
 96 |           return new Worker("testworker", "host", 9200, "/_all", {}, null);
 97 |         }), Error, "invalid number of options");
 98 |       });
 99 |     });
100 |     describe("start", function() {
101 |       it("should establish an http connection using the supplied options", function() {
102 |         new Worker("testworker", "testhost", 9200, "/_all", {
103 |           foo: "bar"
104 |         }, validatorMock).start();
105 |         assert.equal(httpMock.requestOptions.host, "testhost");
106 |         assert.equal(httpMock.requestOptions.port, 9200);
107 |         return assert.equal(httpMock.requestOptions.path, "/_all/_search");
108 |       });
109 |       return it("should send the stringified query through http", function() {
110 |         var queryMock;
111 |         queryMock = {
112 |           foo: "bar"
113 |         };
114 |         new Worker("testworker", "testhost", 9200, "/_all", queryMock, validatorMock).start();
115 |         return assert.equal(httpMock.writeData, JSON.stringify({
116 |           query: queryMock
117 |         }));
118 |       });
119 |     });
120 |     describe("onResponse", function() {
121 |       worker = [][0];
122 |       beforeEach(function() {
123 |         return worker = new Worker("testworker", "testhost", 9200, "/_all", {
124 |           foo: "bar"
125 |         }, validatorMock);
126 |       });
127 |       return it("should emit an 'alarm' event when response status isnt 200", function(done) {
128 |         httpResponseMock.statusCode = 400;
129 |         worker.on("alarm", function(msg) {
130 |           assert.include(msg, Worker.ResultCodes.NotFound.label);
131 |           done();
132 |           return worker.off("alarm");
133 |         });
134 |         return worker.start();
135 |       });
136 |     });
137 |     return describe("handleResponseData", function() {
138 |       var ref, resultStub;
139 |       ref = [], worker = ref[0], resultStub = ref[1];
140 |       beforeEach(function() {
141 |         worker = new Worker("testworker", "testhost", 9200, "/_all", {
142 |           foo: "bar"
143 |         }, validatorMock);
144 |         return resultStub = {
145 |           hits: {
146 |             total: 1,
147 |             hits: [
148 |               {
149 |                 _source: {
150 |                   prop: 1
151 |                 }
152 |               }
153 |             ]
154 |           }
155 |         };
156 |       });
157 |       it("should emit an 'alarm' event when data validation fails due to invalid data", function(done) {
158 |         worker.on("alarm", function(msg) {
159 |           assert.include(msg, Worker.ResultCodes.InvalidResponse.label);
160 |           return done();
161 |         });
162 |         return worker.handleResponseData({});
163 |       });
164 |       it("should emit an 'alarm' event when handleResponseData didn't receive any results", function(done) {
165 |         worker.on("alarm", function(msg) {
166 |           assert.include(msg, Worker.ResultCodes.NoResults.label);
167 |           return done();
168 |         });
169 |         return worker.handleResponseData({
170 |           hits: {
171 |             total: 0,
172 |             hits: []
173 |           }
174 |         });
175 |       });
176 |       it("should emit an 'alarm' event when data validation fails", function(done) {
177 |         validatorMock.validate = (function() {
178 |           return false;
179 |         });
180 |         worker.on("alarm", function(msg) {
181 |           assert.include(msg, Worker.ResultCodes.ValidationFailed.label);
182 |           return done();
183 |         });
184 |         return worker.handleResponseData(resultStub);
185 |       });
186 |       return it("should simply return true if data validation succeeds", function() {
187 |         validatorMock.validate = (function() {
188 |           return true;
189 |         });
190 |         return assert.isTrue(worker.handleResponseData(resultStub));
191 |       });
192 |     });
193 |   });
194 | 
195 | }).call(this);
196 | 


--------------------------------------------------------------------------------
/test_cassandra.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | cqlsh -e "CREATE KEYSPACE test WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor': 1};" --cqlversion="3.4.4"
 4 | cqlsh -e "CREATE TABLE test.TEST (ID TEXT, NAME TEXT, value TEXT, LAST_MODIFIED_DATE TIMESTAMP, PRIMARY KEY (ID));" --cqlversion="3.4.4"
 5 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('2', 'elephant',  '488', toTimestamp(now()));" --cqlversion="3.4.4"
 6 | sleep 1;
 7 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('3', 'elephant',  '598', toTimestamp(now()));" --cqlversion="3.4.4"
 8 | sleep 1;
 9 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('4', 'elephant',  '999', toTimestamp(now()));" --cqlversion="3.4.4"
10 | sleep 1;
11 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('5', 'elephant',  '566', toTimestamp(now()));" --cqlversion="3.4.4"
12 | sleep 1;
13 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('6', 'elephant',  '521', toTimestamp(now()));" --cqlversion="3.4.4"
14 | sleep 1;
15 | cqlsh -e "INSERT INTO test.TEST (ID, NAME, value, LAST_MODIFIED_DATE) VALUES ('7', 'elephant',  '590', toTimestamp(now()));" --cqlversion="3.4.4"
16 | 
17 | bin/queryda --configfile="../jobs/cqlexample.json"
18 | 


--------------------------------------------------------------------------------
/test_elastic.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | ES="127.0.0.1"
 4 | PORT="9200"
 5 | AUTH="admin:elasticFence"
 6 | 
 7 | curl -XPUT "http://$ES:$PORT/monitoring/rum/1" -d '{"requestTime":43,"responseTime":224,"renderTime":568,"timestamp":"2017-09-01T11:47:34"}' -u "$AUTH"
 8 | curl -XPUT "http://$ES:$PORT/monitoring/rum/2" -d '{"requestTime":49,"responseTime":312,"renderTime":619,"timestamp":"2017-09-01T12:02:34"}' -u "$AUTH"
 9 | curl -XPUT "http://$ES:$PORT/monitoring/rum/3" -d '{"requestTime":41,"responseTime":275,"renderTime":597,"timestamp":"2017-09-01T12:17:34"}' -u "$AUTH"
10 | curl -XPUT "http://$ES:$PORT/monitoring/rum/4" -d '{"requestTime":42,"responseTime":301,"renderTime":542,"timestamp":"2017-09-01T12:32:34"}' -u "$AUTH"
11 | curl -XPUT "http://$ES:$PORT/monitoring/rum/5" -d '{"requestTime":48,"responseTime":308,"renderTime":604,"timestamp":"2017-09-01T12:47:34"}' -u "$AUTH"
12 | curl -XPUT "http://$ES:$PORT/monitoring/rum/6" -d '{"requestTime":43,"responseTime":256,"renderTime":531,"timestamp":"2017-09-01T13:02:34"}' -u "$AUTH"
13 | 
14 | bin/queryda --debug --configfile="../jobs/example.json"
15 | 
16 | 


--------------------------------------------------------------------------------