├── .npmignore ├── .gitignore ├── test ├── test-config.json ├── types-test.js ├── helpers.js ├── collector-test.js ├── reduces-test.js ├── metric-test.js ├── event-expression-test.js ├── metric-expression-test.js └── tiers-test.js ├── lib └── cube │ ├── set-immediate.js │ ├── index.js │ ├── bisect.js │ ├── endpoint.js │ ├── emitter.js │ ├── emitter-udp.js │ ├── types.js │ ├── collector.js │ ├── reduces.js │ ├── tiers.js │ ├── emitter-http.js │ ├── database.js │ ├── emitter-ws.js │ ├── collectd.js │ ├── evaluator.js │ ├── server.js │ ├── event-expression.peg │ ├── metric.js │ ├── metric-expression.peg │ └── event.js ├── bin ├── collector.js ├── evaluator.js ├── evaluator-config.js └── collector-config.js ├── Makefile ├── LICENSE ├── examples ├── random-emitter │ ├── random-config.js │ └── random-emitter.js └── event-stream │ ├── event-get.html │ └── event-put.html ├── package.json ├── README.md └── static ├── random └── index.html ├── style.css ├── index.html ├── collectd └── index.html ├── cubism.v1.min.js └── cubism.v1.js /.npmignore: -------------------------------------------------------------------------------- 1 | .git 2 | .gitignore 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .jshintrc 3 | Procfile 4 | -------------------------------------------------------------------------------- /test/test-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "mongo-host": "localhost", 3 | "mongo-port": 27017, 4 | "mongo-database": "cube_test", 5 | "http-port": 1083 6 | } 7 | -------------------------------------------------------------------------------- /lib/cube/set-immediate.js: -------------------------------------------------------------------------------- 1 | if (typeof setImmediate === 'function') { 2 | module.exports = setImmediate; 3 | } else { 4 | module.exports = process.nextTick; 5 | } -------------------------------------------------------------------------------- /lib/cube/index.js: -------------------------------------------------------------------------------- 1 | exports.emitter = require("./emitter"); 2 | exports.server = require("./server"); 3 | exports.collector = require("./collector"); 4 | exports.evaluator = require("./evaluator"); 5 | exports.endpoint = require("./endpoint"); 6 | -------------------------------------------------------------------------------- /lib/cube/bisect.js: -------------------------------------------------------------------------------- 1 | module.exports = bisect; 2 | 3 | function bisect(a, x) { 4 | var lo = 0, hi = a.length; 5 | while (lo < hi) { 6 | var mid = lo + hi >> 1; 7 | if (a[mid] < x) lo = mid + 1; 8 | else hi = mid; 9 | } 10 | return lo; 11 | } 12 | -------------------------------------------------------------------------------- /bin/collector.js: -------------------------------------------------------------------------------- 1 | var options = require("./collector-config"), 2 | cube = require("../"), 3 | server = cube.server(options); 4 | 5 | server.register = function(db, endpoints) { 6 | cube.collector.register(db, endpoints); 7 | }; 8 | 9 | server.start(); 10 | -------------------------------------------------------------------------------- /bin/evaluator.js: -------------------------------------------------------------------------------- 1 | var options = require("./evaluator-config"), 2 | cube = require("../"), 3 | server = cube.server(options); 4 | 5 | server.register = function(db, endpoints) { 6 | cube.evaluator.register(db, endpoints); 7 | }; 8 | 9 | server.start(); 10 | -------------------------------------------------------------------------------- /bin/evaluator-config.js: -------------------------------------------------------------------------------- 1 | // Default configuration for development. 2 | module.exports = { 3 | "mongo-host": "127.0.0.1", 4 | "mongo-port": 27017, 5 | "mongo-database": "cube_development", 6 | "mongo-username": null, 7 | "mongo-password": null, 8 | "http-port": 1081 9 | }; 10 | -------------------------------------------------------------------------------- /bin/collector-config.js: -------------------------------------------------------------------------------- 1 | // Default configuration for development. 2 | module.exports = { 3 | "mongo-host": "127.0.0.1", 4 | "mongo-port": 27017, 5 | "mongo-database": "cube_development", 6 | "mongo-username": null, 7 | "mongo-password": null, 8 | "http-port": 1080, 9 | "udp-port": 1180 10 | }; 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | JS_TESTER = ./node_modules/vows/bin/vows 2 | PEG_COMPILER = ./node_modules/pegjs/bin/pegjs 3 | 4 | .PHONY: test 5 | 6 | %.js: %.peg Makefile 7 | $(PEG_COMPILER) < $< > $@ 8 | 9 | all: \ 10 | lib/cube/event-expression.js \ 11 | lib/cube/metric-expression.js 12 | 13 | test: all 14 | @$(JS_TESTER) 15 | -------------------------------------------------------------------------------- /lib/cube/endpoint.js: -------------------------------------------------------------------------------- 1 | // creates an endpoint with given HTTP method, URL path and dispatch (function) 2 | // (method argument is optional) 3 | // endpoints are evaluated in server.js and 4 | // dispatch(request, response) is called if path/method matches 5 | module.exports = function(method, path, dispatch) { 6 | return { 7 | match: arguments.length < 3 8 | ? (dispatch = path, path = method, function(p) { return p == path; }) 9 | : function(p, m) { return m == method && p == path; }, 10 | dispatch: dispatch 11 | }; 12 | } 13 | -------------------------------------------------------------------------------- /lib/cube/emitter.js: -------------------------------------------------------------------------------- 1 | var url = require("url"), 2 | http = require("./emitter-http"), 3 | udp = require("./emitter-udp"), 4 | ws = require("./emitter-ws"); 5 | 6 | // returns an emmiter for the given URL; handles http://, udp:// or ws:// protocols 7 | module.exports = function(u) { 8 | var emitter; 9 | u = url.parse(u); 10 | switch (u.protocol) { 11 | case "udp:": emitter = udp; break; 12 | case "ws:": case "wss:": emitter = ws; break; 13 | case "http:": emitter = http; break; 14 | } 15 | return emitter(u.protocol, u.hostname, u.port); 16 | }; 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2011-2014 Square, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 4 | this file except in compliance with the License. You may obtain a copy of the 5 | License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software distributed 10 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | specific language governing permissions and limitations under the License. 13 | -------------------------------------------------------------------------------- /examples/random-emitter/random-config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 3 | // The collector to send events to. 4 | "collector": "ws://127.0.0.1:1080", 5 | 6 | // The offset and duration to backfill, in milliseconds. 7 | // For example, if the offset is minus four hours, then the first event that 8 | // the random emitter sends will be four hours old. It will then generate more 9 | // recent events based on the step interval, all the way up to the duration. 10 | "offset": -4 * 60 * 60 * 1000, 11 | "duration": 8 * 60 * 60 * 1000, 12 | 13 | // The time between random events. 14 | "step": 1000 * 10 15 | }; 16 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cube", 3 | "version": "0.2.12", 4 | "description": "A system for analyzing time series data using MongoDB and Node.", 5 | "keywords": [ 6 | "time series" 7 | ], 8 | "homepage": "http://square.github.com/cube/", 9 | "author": { 10 | "name": "Mike Bostock", 11 | "url": "http://bost.ocks.org/mike" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "http://github.com/square/cube.git" 16 | }, 17 | "main": "./lib/cube", 18 | "dependencies": { 19 | "mongodb": "~1.3.18", 20 | "node-static": "0.6.5", 21 | "pegjs": "0.7.0", 22 | "vows": "0.7.0", 23 | "websocket": "1.0.8", 24 | "websocket-server": "1.4.04" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /examples/random-emitter/random-emitter.js: -------------------------------------------------------------------------------- 1 | process.env.TZ = 'UTC'; 2 | 3 | var util = require("util"), 4 | cube = require("../../"), // replace with require("cube") 5 | options = require("./random-config"); 6 | 7 | util.log("starting emitter"); 8 | var emitter = cube.emitter(options["collector"]); 9 | 10 | var start = Date.now() + options["offset"], 11 | stop = start + options["duration"], 12 | step = options["step"], 13 | value = 0, 14 | count = 0; 15 | 16 | while (start < stop) { 17 | emitter.send({ 18 | type: "random", 19 | time: new Date(start), 20 | data: { 21 | value: value += Math.random() - .5 22 | } 23 | }); 24 | start += step; 25 | ++count; 26 | } 27 | 28 | util.log("sent " + count + " events"); 29 | util.log("stopping emitter"); 30 | emitter.close(); 31 | -------------------------------------------------------------------------------- /examples/event-stream/event-get.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

Streaming Events - Get

4 | 5 |

This page streams events from Cube's evaluator (the /event/get endpoint) and logs them to the JavaScript console. If you open event-put.html in a new window, you'll start receiving events. 6 | 7 | 33 | -------------------------------------------------------------------------------- /examples/event-stream/event-put.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

Streaming Events - Put

4 | 5 |

This page streams events to Cube's collector (the /event/put endpoint) and logs them to the JavaScript console. If you open event-get.html in a new window, you can verify that Cube is receiving the events. 6 | 7 | 39 | -------------------------------------------------------------------------------- /lib/cube/emitter-udp.js: -------------------------------------------------------------------------------- 1 | var util = require("util"), 2 | dgram = require("dgram"), 3 | setImmediate = require("./set-immediate"); 4 | 5 | // returns an emitter which sneds events one at a time to the given udp://host:port 6 | module.exports = function(protocol, host, port) { 7 | var emitter = {}, 8 | queue = [], 9 | udp = dgram.createSocket("udp4"), 10 | closing; 11 | 12 | if (protocol != "udp:") throw new Error("invalid UDP protocol"); 13 | 14 | function send() { 15 | var event = queue.pop(); 16 | if (!event) return; 17 | var buffer = new Buffer(JSON.stringify(event)); 18 | udp.send(buffer, 0, buffer.length, port, host, function(error) { 19 | if (error) console.warn(error); 20 | if (queue.length) setImmediate(send); 21 | else if (closing) udp.close(); 22 | }); 23 | } 24 | 25 | emitter.send = function(event) { 26 | if (!closing && queue.push(event) == 1) setImmediate(send); 27 | return emitter; 28 | }; 29 | 30 | emitter.close = function() { 31 | if (queue.length) closing = 1; 32 | else udp.close(); 33 | return emitter; 34 | }; 35 | 36 | return emitter; 37 | }; 38 | -------------------------------------------------------------------------------- /lib/cube/types.js: -------------------------------------------------------------------------------- 1 | // Much like db.collection, but caches the result for both events and metrics. 2 | // Also, this is synchronous, since we are opening a collection unsafely. 3 | var types = module.exports = function(db) { 4 | var collections = {}; 5 | return function(type) { 6 | var collection = collections[type]; 7 | if (!collection) { 8 | collection = collections[type] = {}; 9 | db.collection(type + "_events", function(error, events) { 10 | collection.events = events; 11 | }); 12 | db.collection(type + "_metrics", function(error, metrics) { 13 | collection.metrics = metrics; 14 | }); 15 | } 16 | return collection; 17 | }; 18 | }; 19 | 20 | var eventRe = /_events$/; 21 | 22 | types.getter = function(db) { 23 | return function(request, callback) { 24 | db.collectionNames(function(error, names) { 25 | handle(error); 26 | callback(names 27 | .map(function(d) { return d.name.split(".")[1]; }) 28 | .filter(function(d) { return eventRe.test(d); }) 29 | .map(function(d) { return d.substring(0, d.length - 7); }) 30 | .sort()); 31 | }); 32 | }; 33 | }; 34 | 35 | function handle(error) { 36 | if (error) throw error; 37 | } 38 | -------------------------------------------------------------------------------- /lib/cube/collector.js: -------------------------------------------------------------------------------- 1 | var endpoint = require("./endpoint"); 2 | 3 | // 4 | var headers = { 5 | "Content-Type": "application/json", 6 | "Access-Control-Allow-Origin": "*" 7 | }; 8 | 9 | exports.register = function(db, endpoints) { 10 | var putter = require("./event").putter(db), 11 | poster = post(putter); 12 | 13 | // 14 | endpoints.ws.push( 15 | endpoint("/1.0/event/put", putter) 16 | ); 17 | 18 | // 19 | endpoints.http.push( 20 | endpoint("POST", "/1.0/event", poster), 21 | endpoint("POST", "/1.0/event/put", poster), 22 | endpoint("POST", "/collectd", require("./collectd").putter(putter)) 23 | ); 24 | 25 | // 26 | endpoints.udp = putter; 27 | }; 28 | 29 | function post(putter) { 30 | return function(request, response) { 31 | var content = ""; 32 | request.on("data", function(chunk) { 33 | content += chunk; 34 | }); 35 | request.on("end", function() { 36 | try { 37 | JSON.parse(content).forEach(putter); 38 | } catch (e) { 39 | response.writeHead(400, headers); 40 | response.end(JSON.stringify({error: e.toString()})); 41 | return; 42 | } 43 | response.writeHead(200, headers); 44 | response.end("{}"); 45 | }); 46 | }; 47 | } 48 | -------------------------------------------------------------------------------- /test/types-test.js: -------------------------------------------------------------------------------- 1 | var vows = require("vows"), 2 | assert = require("assert"), 3 | test = require("./helpers"), 4 | types = require("../lib/cube/types"), 5 | mongodb = require("mongodb"); 6 | 7 | var suite = vows.describe("types"); 8 | 9 | suite.addBatch(test.batch({ 10 | topic: function(test) { 11 | return types(test.db); 12 | }, 13 | 14 | "types": { 15 | "returns collection cache for a given database": function(types) { 16 | assert.equal(typeof types, "function"); 17 | }, 18 | "each typed collection has events and metrics": function(types) { 19 | var collection = types("random"), 20 | keys = []; 21 | for (var key in collection) { 22 | keys.push(key); 23 | } 24 | keys.sort(); 25 | assert.deepEqual(keys, ["events", "metrics"]); 26 | assert.isTrue(collection.events instanceof mongodb.Collection); 27 | assert.isTrue(collection.metrics instanceof mongodb.Collection); 28 | assert.equal(collection.events.collectionName, "random_events"); 29 | assert.equal(collection.metrics.collectionName, "random_metrics"); 30 | }, 31 | "memoizes cached collections": function(types) { 32 | assert.strictEqual(types("random"), types("random")); 33 | } 34 | } 35 | })); 36 | 37 | suite.export(module); 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cube 2 | 3 | **Cube** is a system for collecting timestamped events and deriving metrics. By collecting events rather than metrics, Cube lets you compute aggregate statistics *post hoc*. It also enables richer analysis, such as quantiles and histograms of arbitrary event sets. Cube is built on [MongoDB](http://www.mongodb.org) and available under the [Apache License](/square/cube/blob/master/LICENSE). 4 | 5 | Want to learn more? [See the wiki.](https://github.com/square/cube/wiki) 6 | 7 | ## Status 8 | 9 | Cube is **not under active development, maintenance or support by Square** (or by its original author Mike Bostock). It has been deprecated internally for over a year. We keep it running for historical interest because it powers some interesting visualizations, but new production systems have replaced it for analytics purposes. 10 | 11 | [Infochimps](https://github.com/infochimps-labs/cube) worked on a fork of Cube which diverged slightly from the Square version. Github user [Marsup](https://github.com/marsup/cube) has been working to [merge the two versions](https://github.com/square/cube/pull/129) with some success, but there are no plans to complete the merge or publish new versions under the original Square repository or npm package. 12 | 13 | Please use the [cube-user](https://groups.google.com/forum/#!forum/cube-user) list on Google Groups for all further discussion of the Cube project. 14 | -------------------------------------------------------------------------------- /lib/cube/reduces.js: -------------------------------------------------------------------------------- 1 | var reduces = module.exports = { 2 | 3 | sum: function(values) { 4 | var i = -1, n = values.length, sum = 0; 5 | while (++i < n) sum += values[i]; 6 | return sum; 7 | }, 8 | 9 | min: function(values) { 10 | var i = -1, n = values.length, min = Infinity, value; 11 | while (++i < n) if ((value = values[i]) < min) min = value; 12 | return min; 13 | }, 14 | 15 | max: function(values) { 16 | var i = -1, n = values.length, max = -Infinity, value; 17 | while (++i < n) if ((value = values[i]) > max) max = value; 18 | return max; 19 | }, 20 | 21 | distinct: function(values) { 22 | var map = {}, count = 0, i = -1, n = values.length, value; 23 | while (++i < n) if (!((value = values[i]) in map)) map[value] = ++count; 24 | return count; 25 | }, 26 | 27 | median: function(values) { 28 | return quantile(values.sort(ascending), .5); 29 | } 30 | 31 | }; 32 | 33 | // These metrics have well-defined values for the empty set. 34 | reduces.sum.empty = 0; 35 | reduces.distinct.empty = 0; 36 | 37 | // These metrics can be computed using pyramidal aggregation. 38 | reduces.sum.pyramidal = true; 39 | reduces.min.pyramidal = true; 40 | reduces.max.pyramidal = true; 41 | 42 | function ascending(a, b) { 43 | return a - b; 44 | } 45 | 46 | function quantile(values, q) { 47 | var i = 1 + q * (values.length - 1), 48 | j = ~~i, 49 | h = i - j, 50 | a = values[j - 1]; 51 | return h ? a + h * (values[j] - a) : a; 52 | } 53 | -------------------------------------------------------------------------------- /lib/cube/tiers.js: -------------------------------------------------------------------------------- 1 | var tiers = module.exports = {}; 2 | 3 | var second = 1000, 4 | second10 = 10 * second, 5 | minute = 60 * second, 6 | minute5 = 5 * minute, 7 | hour = 60 * minute, 8 | day = 24 * hour; 9 | 10 | tiers[second10] = { 11 | key: second10, 12 | floor: function(d) { return new Date(Math.floor(d / second10) * second10); }, 13 | ceil: tier_ceil, 14 | step: function(d) { return new Date(+d + second10); } 15 | }; 16 | 17 | tiers[minute] = { 18 | key: minute, 19 | floor: function(d) { return new Date(Math.floor(d / minute) * minute); }, 20 | ceil: tier_ceil, 21 | step: function(d) { return new Date(+d + minute); } 22 | }; 23 | 24 | tiers[minute5] = { 25 | key: minute5, 26 | floor: function(d) { return new Date(Math.floor(d / minute5) * minute5); }, 27 | ceil: tier_ceil, 28 | step: function(d) { return new Date(+d + minute5); } 29 | }; 30 | 31 | tiers[hour] = { 32 | key: hour, 33 | floor: function(d) { return new Date(Math.floor(d / hour) * hour); }, 34 | ceil: tier_ceil, 35 | step: function(d) { return new Date(+d + hour); }, 36 | next: tiers[minute5], 37 | size: function() { return 12; } 38 | }; 39 | 40 | tiers[day] = { 41 | key: day, 42 | floor: function(d) { return new Date(Math.floor(d / day) * day); }, 43 | ceil: tier_ceil, 44 | step: function(d) { return new Date(+d + day); }, 45 | next: tiers[hour], 46 | size: function() { return 24; } 47 | }; 48 | 49 | function tier_ceil(date) { 50 | return this.step(this.floor(new Date(date - 1))); 51 | } 52 | -------------------------------------------------------------------------------- /test/helpers.js: -------------------------------------------------------------------------------- 1 | var database = require("../lib/cube/database"), 2 | util = require("util"), 3 | http = require("http"); 4 | 5 | var config = exports.config = require('./test-config'); 6 | 7 | exports.batch = function(batch) { 8 | return { 9 | "": { 10 | topic: function() { 11 | var cb = this.callback; 12 | database.open(config, function(error, db) { 13 | if (error) { 14 | return cb(error); 15 | } 16 | var collectionsRemaining = 2; 17 | db.dropCollection("test_events", collectionReady); 18 | db.dropCollection("test_metrics", collectionReady); 19 | function collectionReady() { 20 | if (!--collectionsRemaining) { 21 | cb(null, {db: db}); 22 | } 23 | } 24 | }); 25 | }, 26 | "": batch, 27 | teardown: function(test) { 28 | test.db.close(); 29 | } 30 | } 31 | }; 32 | }; 33 | 34 | exports.request = function(options, data) { 35 | return function() { 36 | var cb = this.callback; 37 | 38 | options.host = "localhost"; 39 | 40 | var request = http.request(options, function(response) { 41 | response.body = ""; 42 | response.setEncoding("utf8"); 43 | response.on("data", function(chunk) { response.body += chunk; }); 44 | response.on("end", function() { cb(null, response); }); 45 | }); 46 | 47 | request.on("error", function(e) { cb(e, null); }); 48 | 49 | if (data && data.length > 0) request.write(data); 50 | request.end(); 51 | }; 52 | }; 53 | 54 | // Disable logging for tests. 55 | util.log = function() {}; 56 | -------------------------------------------------------------------------------- /lib/cube/emitter-http.js: -------------------------------------------------------------------------------- 1 | var util = require("util"), 2 | http = require("http"); 3 | 4 | var batchSize = 500, // events per batch 5 | batchInterval = 500, // ms between batches 6 | errorInterval = 1000; // ms between retries 7 | 8 | // returns an emitter which POSTs events up to 500 at a time to the given http://host:port 9 | module.exports = function(protocol, host, port) { 10 | var emitter = {}, 11 | queue = [], 12 | closing; 13 | 14 | if (protocol != "http:") throw new Error("invalid HTTP protocol"); 15 | 16 | function send() { 17 | var events = queue.splice(0, batchSize), 18 | body = JSON.stringify(events); 19 | 20 | http.request({ 21 | host: host, 22 | port: port, 23 | path: "/1.0/event", 24 | method: "POST", 25 | headers: { 26 | "Content-Type": "application/json", 27 | "Content-Length": body.length 28 | } 29 | }, function(response) { 30 | if (response.statusCode !== 200) return error(response.statusCode); 31 | if (queue.length) setTimeout(send, batchInterval); 32 | }).on("error", function(e) { 33 | error(e.message); 34 | }).end(body); 35 | 36 | function error(message) { 37 | util.log("error: " + message); 38 | queue.unshift.apply(queue, events); 39 | setTimeout(send, errorInterval); 40 | } 41 | } 42 | 43 | emitter.send = function(event) { 44 | if (!closing && queue.push(event) === 1) setTimeout(send, batchInterval); 45 | return emitter; 46 | }; 47 | 48 | emitter.close = function () { 49 | if (queue.length) closing = 1; 50 | return emitter; 51 | }; 52 | 53 | return emitter; 54 | }; 55 | -------------------------------------------------------------------------------- /lib/cube/database.js: -------------------------------------------------------------------------------- 1 | var mongodb = require("mongodb"); 2 | 3 | var database = module.exports = {}; 4 | 5 | // Opens MongoDB driver given connection URL and optional options: 6 | // 7 | // { 8 | // "mongo-url": "", 9 | // "mongo-options": { 10 | // "db": { "safe": false }, 11 | // "server": { "auto_reconnect": true }, 12 | // "replSet": { "read_secondary": true } 13 | // } 14 | // } 15 | // 16 | // See http://docs.mongodb.org/manual/reference/connection-string/ for details. 17 | // You can also specify a Replica Set this way. 18 | // 19 | database.open = function(config, callback) { 20 | var url = config["mongo-url"] || database.config2url(config), 21 | options = config["mongo-options"] || database.config2options(config); 22 | return mongodb.Db.connect(url, options, callback); 23 | }; 24 | 25 | // 26 | // For backwards-compatibility you can specify a connection to a single Mongo(s) as follows: 27 | // 28 | // { 29 | // "mongo-host": "localhost", 30 | // "mongo-port": "27017", 31 | // "mongo-server-options": { "auto_reconnect": true }, 32 | // "mongo-database": "cube", 33 | // "mongo-database-options": { "safe": false }, 34 | // "mongo-username": null, 35 | // "mongo-password": null, 36 | // } 37 | // (defaults are shown) 38 | // 39 | database.config2url = function(config) { 40 | var user = config["mongo-username"], 41 | pass = config["mongo-password"], 42 | host = config["mongo-host"] || "localhost", 43 | port = config["mongo-port"] || 27017, 44 | name = config["mongo-database"] || "cube", 45 | auth = user ? user+":"+pass+"@" : ""; 46 | return "mongodb://"+auth+host+":"+port+"/"+name; 47 | }; 48 | 49 | database.config2options = function(config) { 50 | return { 51 | db: config["mongo-database-options"] || { safe: false }, 52 | server: config["mongo-server-options"] || { auto_reconnect: true }, 53 | replSet: { read_secondary: true } 54 | }; 55 | }; -------------------------------------------------------------------------------- /static/random/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Random 4 | 9 | 14 | 15 | 16 | 76 | -------------------------------------------------------------------------------- /lib/cube/emitter-ws.js: -------------------------------------------------------------------------------- 1 | var util = require("util"), 2 | websocket = require("websocket"); 3 | 4 | // returns an emitter which sends events one at a time to the given ws://host:port 5 | module.exports = function(protocol, host, port) { 6 | var emitter = {}, 7 | queue = [], 8 | url = protocol + "//" + host + ":" + port + "/1.0/event/put", 9 | socket, 10 | timeout, 11 | closing; 12 | 13 | function close() { 14 | if (socket) { 15 | util.log("closing socket"); 16 | socket.removeListener("error", reopen); 17 | socket.removeListener("close", reopen); 18 | socket.close(); 19 | socket = null; 20 | } 21 | } 22 | 23 | function closeWhenDone() { 24 | closing = true; 25 | if (socket) { 26 | if (!socket.bytesWaitingToFlush) close(); 27 | else setTimeout(closeWhenDone, 1000); 28 | } 29 | } 30 | 31 | function open() { 32 | timeout = 0; 33 | close(); 34 | util.log("opening socket: " + url); 35 | var client = new websocket.client(); 36 | client.on("connect", function(connection) { 37 | socket = connection; 38 | socket.on("message", log); 39 | socket.on("error", reopen); 40 | socket.on("close", reopen); 41 | flush(); 42 | if (closing) closeWhenDone(); 43 | }); 44 | client.on("connectFailed", reopen); 45 | client.on("error", reopen); 46 | client.connect(url); 47 | } 48 | 49 | function reopen() { 50 | if (!timeout && !closing) { 51 | util.log("reopening soon"); 52 | timeout = setTimeout(open, 1000); 53 | } 54 | } 55 | 56 | function flush() { 57 | var event; 58 | while (event = queue.pop()) { 59 | try { 60 | socket.sendUTF(JSON.stringify(event)); 61 | } catch (e) { 62 | util.log(e.stack); 63 | reopen(); 64 | return queue.push(event); 65 | } 66 | } 67 | } 68 | 69 | function log(message) { 70 | util.log(message.utf8Data); 71 | } 72 | 73 | emitter.send = function(event) { 74 | queue.push(event); 75 | if (socket) flush(); 76 | return emitter; 77 | }; 78 | 79 | emitter.close = function() { 80 | closeWhenDone(); 81 | return emitter; 82 | }; 83 | 84 | open(); 85 | 86 | return emitter; 87 | }; 88 | -------------------------------------------------------------------------------- /lib/cube/collectd.js: -------------------------------------------------------------------------------- 1 | exports.putter = function(putter) { 2 | var valuesByKey = {}; 3 | 4 | // Converts a collectd value list to a Cube event. 5 | function event(values) { 6 | var root = {host: values.host}, 7 | data = root, 8 | parent, 9 | key; 10 | 11 | // The plugin and type are required. If the type is the same as the plugin, 12 | // then ignore the type (for example, memory/memory and load/load). 13 | parent = data, data = data[key = values.plugin] || (data[values.plugin] = {}); 14 | if (values.type != values.plugin) parent = data, data = data[key = values.type] || (data[values.type] = {}); 15 | 16 | // The plugin_instance and type_instance are optional. 17 | if (values.plugin_instance) root.plugin = values.plugin_instance; 18 | if (values.type_instance) root.type = values.type_instance; 19 | 20 | // If only a single value is specified, then don't store a map of named 21 | // values; just store the single value using the type_instance name (e.g., 22 | // memory/memory-inactive, df-root/df_complex-used). Otherwise, iterate over 23 | // the received values and store them as a map. 24 | if (values.values.length == 1) parent[key] = value(0); 25 | else values.dsnames.forEach(function(d, i) { data[d] = value(i); }); 26 | 27 | // For "derive" events, we must compute the delta since the last event. 28 | function value(i) { 29 | var d = values.values[i]; 30 | switch (values.dstypes[i]) { 31 | case "derive": { 32 | var key = values.host + "/" + values.plugin + "/" + values.plugin_instance + "/" + values.type + "/" + values.type_instance + "/" + values.dsnames[i], 33 | value = key in valuesByKey ? valuesByKey[key] : d; 34 | valuesByKey[key] = d; 35 | d -= value; 36 | break; 37 | } 38 | } 39 | return d; 40 | } 41 | 42 | return { 43 | type: "collectd", 44 | time: new Date(+values.time), 45 | data: root 46 | }; 47 | } 48 | 49 | return function(request, response) { 50 | var content = ""; 51 | request.on("data", function(chunk) { 52 | content += chunk; 53 | }); 54 | request.on("end", function() { 55 | var future = Date.now() / 1e3 + 1e9; 56 | JSON.parse(content).forEach(function(values) { 57 | var time = values.time; 58 | if (time > future) time /= 1073741824; 59 | values.time = Math.round(time) * 1e3; 60 | putter(event(values)); 61 | }); 62 | response.writeHead(200); 63 | response.end(); 64 | }); 65 | }; 66 | }; 67 | -------------------------------------------------------------------------------- /static/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: "Helvetica Neue", Helvetica, sans-serif; 3 | margin: 30px auto; 4 | width: 1440px; 5 | position: relative; 6 | } 7 | 8 | header { 9 | font-weight: 500; 10 | padding: 6px; 11 | } 12 | 13 | .group { 14 | margin-bottom: 1em; 15 | } 16 | 17 | .axis { 18 | font: 10px sans-serif; 19 | position: fixed; 20 | pointer-events: none; 21 | z-index: 2; 22 | } 23 | 24 | .axis text { 25 | -webkit-transition: fill-opacity 250ms linear; 26 | } 27 | 28 | .axis path { 29 | display: none; 30 | } 31 | 32 | .axis line { 33 | stroke: #000; 34 | shape-rendering: crispEdges; 35 | } 36 | 37 | .axis.top { 38 | background-image: linear-gradient(top, #fff 0%, rgba(255,255,255,0) 100%); 39 | background-image: -o-linear-gradient(top, #fff 0%, rgba(255,255,255,0) 100%); 40 | background-image: -moz-linear-gradient(top, #fff 0%, rgba(255,255,255,0) 100%); 41 | background-image: -webkit-linear-gradient(top, #fff 0%, rgba(255,255,255,0) 100%); 42 | background-image: -ms-linear-gradient(top, #fff 0%, rgba(255,255,255,0) 100%); 43 | top: 0px; 44 | padding: 0 0 24px 0; 45 | } 46 | 47 | .axis.bottom { 48 | background-image: linear-gradient(bottom, #fff 0%, rgba(255,255,255,0) 100%); 49 | background-image: -o-linear-gradient(bottom, #fff 0%, rgba(255,255,255,0) 100%); 50 | background-image: -moz-linear-gradient(bottom, #fff 0%, rgba(255,255,255,0) 100%); 51 | background-image: -webkit-linear-gradient(bottom, #fff 0%, rgba(255,255,255,0) 100%); 52 | background-image: -ms-linear-gradient(bottom, #fff 0%, rgba(255,255,255,0) 100%); 53 | bottom: 0px; 54 | padding: 24px 0 0 0; 55 | } 56 | 57 | .horizon { 58 | border-bottom: solid 1px #000; 59 | overflow: hidden; 60 | position: relative; 61 | } 62 | 63 | :not(.horizon) + .horizon { 64 | border-top: solid 1px #000; 65 | } 66 | 67 | .horizon canvas { 68 | display: block; 69 | } 70 | 71 | .horizon .title, 72 | .horizon .value { 73 | bottom: 0; 74 | line-height: 30px; 75 | margin: 0 6px; 76 | position: absolute; 77 | text-shadow: 0 1px 0 rgba(255,255,255,.5); 78 | white-space: nowrap; 79 | } 80 | 81 | .horizon .title { 82 | left: 0; 83 | } 84 | 85 | .horizon .value { 86 | right: 0; 87 | } 88 | 89 | .line { 90 | background: #000; 91 | z-index: 1; 92 | } 93 | 94 | #step { 95 | position: fixed; 96 | bottom: 6px; 97 | z-index: 3; 98 | } 99 | 100 | @media all and (max-width: 1439px) { 101 | body { margin: 0px auto; } 102 | .axis { position: static; } 103 | .axis.top, .axis.bottom { padding: 0; } 104 | } 105 | -------------------------------------------------------------------------------- /static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Cube 4 | 9 | 14 | 15 | 16 | 87 | -------------------------------------------------------------------------------- /test/collector-test.js: -------------------------------------------------------------------------------- 1 | var vows = require("vows"), 2 | assert = require("assert"), 3 | cube = require("../"), 4 | test = require("./helpers"); 5 | 6 | var suite = vows.describe("collector"); 7 | 8 | var server = cube.server(test.config), 9 | port = test.config["http-port"]; 10 | 11 | console.log('collector port %s', port); 12 | 13 | server.register = cube.collector.register; 14 | 15 | server.start(); 16 | 17 | suite.addBatch(test.batch({ 18 | "POST /event/put with invalid JSON": { 19 | topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, "This ain't JSON.\n"), 20 | "responds with status 400": function(response) { 21 | assert.equal(response.statusCode, 400); 22 | assert.deepEqual(JSON.parse(response.body), {error: "SyntaxError: Unexpected token T"}); 23 | } 24 | } 25 | })); 26 | 27 | suite.addBatch(test.batch({ 28 | "POST /event/put with a JSON object": { 29 | topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify({ 30 | type: "test", 31 | time: new Date(), 32 | data: { 33 | foo: "bar" 34 | } 35 | })), 36 | "responds with status 400": function(response) { 37 | assert.equal(response.statusCode, 400); 38 | assert.deepEqual(JSON.parse(response.body), {error: "TypeError: Object # has no method 'forEach'"}); 39 | } 40 | } 41 | })); 42 | 43 | suite.addBatch(test.batch({ 44 | "POST /event/put with a JSON array": { 45 | topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify([{ 46 | type: "test", 47 | time: new Date(), 48 | data: { 49 | foo: "bar" 50 | } 51 | }])), 52 | "responds with status 200": function(response) { 53 | assert.equal(response.statusCode, 200); 54 | assert.deepEqual(JSON.parse(response.body), {}); 55 | } 56 | } 57 | })); 58 | 59 | suite.addBatch(test.batch({ 60 | "POST /event/put with a JSON number": { 61 | topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify(42)), 62 | "responds with status 400": function(response) { 63 | assert.equal(response.statusCode, 400); 64 | assert.deepEqual(JSON.parse(response.body), {error: "TypeError: Object 42 has no method 'forEach'"}); 65 | } 66 | } 67 | })); 68 | 69 | suite.addBatch(test.batch({ 70 | "POST /event/put without an associated time": { 71 | topic: test.request({method: "POST", port: port, path: "/1.0/event/put"}, JSON.stringify([{ 72 | type: "test", 73 | data: { 74 | foo: "bar" 75 | } 76 | }])), 77 | "responds with status 200": function(response) { 78 | assert.equal(response.statusCode, 200); 79 | assert.deepEqual(JSON.parse(response.body), {}); 80 | } 81 | } 82 | })); 83 | 84 | suite.export(module); 85 | -------------------------------------------------------------------------------- /static/collectd/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Collectd 4 | 9 | 14 | 15 | 16 | 87 | -------------------------------------------------------------------------------- /lib/cube/evaluator.js: -------------------------------------------------------------------------------- 1 | var endpoint = require("./endpoint"), 2 | url = require("url"); 3 | 4 | // To avoid running out of memory, the GET endpoints have a maximum number of 5 | // values they can return. If the limit is exceeded, only the most recent 6 | // results are returned. 7 | var limitMax = 1e4; 8 | 9 | // 10 | var headers = { 11 | "Content-Type": "application/json", 12 | "Access-Control-Allow-Origin": "*" 13 | }; 14 | 15 | exports.register = function(db, endpoints) { 16 | var event = require("./event").getter(db), 17 | metric = require("./metric").getter(db), 18 | types = require("./types").getter(db); 19 | 20 | // 21 | endpoints.ws.push( 22 | endpoint("/1.0/event/get", event), 23 | endpoint("/1.0/metric/get", metric), 24 | endpoint("/1.0/types/get", types) 25 | ); 26 | 27 | // 28 | endpoints.http.push( 29 | endpoint("GET", "/1.0/event", eventGet), 30 | endpoint("GET", "/1.0/event/get", eventGet), 31 | endpoint("GET", "/1.0/metric", metricGet), 32 | endpoint("GET", "/1.0/metric/get", metricGet), 33 | endpoint("GET", "/1.0/types", typesGet), 34 | endpoint("GET", "/1.0/types/get", typesGet) 35 | ); 36 | 37 | function eventGet(request, response) { 38 | request = url.parse(request.url, true).query; 39 | 40 | var data = []; 41 | 42 | // Provide default start and stop times for recent events. 43 | // If the limit is not specified, or too big, use the maximum limit. 44 | if (!("stop" in request)) request.stop = Date.now(); 45 | if (!("start" in request)) request.start = 0; 46 | if (!(+request.limit <= limitMax)) request.limit = limitMax; 47 | 48 | if (event(request, callback) < 0) { 49 | response.writeHead(400, headers); 50 | response.end(JSON.stringify(data[0])); 51 | } else { 52 | response.writeHead(200, headers); 53 | } 54 | 55 | function callback(d) { 56 | if (d == null) response.end(JSON.stringify(data.reverse())); 57 | else data.push(d); 58 | } 59 | } 60 | 61 | function metricGet(request, response) { 62 | request = url.parse(request.url, true).query; 63 | 64 | var data = [], 65 | limit = +request.limit, 66 | step = +request.step; 67 | 68 | // Provide default start, stop and step times for recent metrics. 69 | // If the limit is not specified, or too big, use the maximum limit. 70 | if (!("step" in request)) request.step = step = 1e4; 71 | if (!("stop" in request)) request.stop = Math.floor(Date.now() / step) * step; 72 | if (!("start" in request)) request.start = 0; 73 | if (!(limit <= limitMax)) limit = limitMax; 74 | 75 | // If the time between start and stop is too long, then bring the start time 76 | // forward so that only the most recent results are returned. This is only 77 | // approximate in the case of months, but why would you want to return 78 | // exactly ten thousand months? Don't rely on exact limits! 79 | var start = new Date(request.start), 80 | stop = new Date(request.stop); 81 | if ((stop - start) / step > limit) request.start = new Date(stop - step * limit); 82 | 83 | if (metric(request, callback) < 0) { 84 | response.writeHead(400, headers); 85 | response.end(JSON.stringify(data[0])); 86 | } else { 87 | response.writeHead(200, headers); 88 | } 89 | 90 | function callback(d) { 91 | if (d.time >= stop) response.end(JSON.stringify(data.sort(chronological))); 92 | else data.push(d); 93 | } 94 | } 95 | 96 | function typesGet(request, response) { 97 | types(url.parse(request.url, true).query, function(data) { 98 | response.writeHead(200, headers); 99 | response.end(JSON.stringify(data)); 100 | }); 101 | } 102 | }; 103 | 104 | function chronological(a, b) { 105 | return a.time - b.time; 106 | } 107 | -------------------------------------------------------------------------------- /test/reduces-test.js: -------------------------------------------------------------------------------- 1 | var vows = require("vows"), 2 | assert = require("assert"), 3 | reduces = require("../lib/cube/reduces"); 4 | 5 | var suite = vows.describe("reduces"); 6 | 7 | suite.addBatch({ 8 | "reduces": { 9 | "contains exactly the expected reduces": function() { 10 | var keys = []; 11 | for (var key in reduces) { 12 | keys.push(key); 13 | } 14 | keys.sort(); 15 | assert.deepEqual(keys, ["distinct", "max", "median", "min", "sum"]); 16 | } 17 | }, 18 | 19 | "distinct": { 20 | topic: function() { 21 | return reduces.distinct; 22 | }, 23 | "empty is zero": function(reduce) { 24 | assert.strictEqual(reduce.empty, 0); 25 | }, 26 | "is not pyramidal": function(reduce) { 27 | assert.isTrue(!reduce.pyramidal); 28 | }, 29 | "returns the number of distinct values": function(reduce) { 30 | assert.equal(reduce([1, 2, 3, 2, 1]), 3); 31 | }, 32 | "determines uniqueness based on string coercion": function(reduce) { 33 | assert.equal(reduce([{}, {}, {}]), 1); 34 | assert.equal(reduce([{}, "[object Object]", new Object]), 1); 35 | assert.equal(reduce([new Number(1), 1, "1"]), 1); 36 | assert.equal(reduce([new Number(1), 2, "3", "2", 3, new Number(1)]), 3); 37 | assert.equal(reduce([{toString: function() { return 1; }}, 1, 2]), 2); 38 | } 39 | }, 40 | 41 | "max": { 42 | topic: function() { 43 | return reduces.max; 44 | }, 45 | "empty is undefined": function(reduce) { 46 | assert.strictEqual(reduce.empty, undefined); 47 | }, 48 | "is pyramidal": function(reduce) { 49 | assert.isTrue(reduce.pyramidal); 50 | }, 51 | "returns the maximum value": function(reduce) { 52 | assert.equal(reduce([1, 2, 3, 2, 1]), 3); 53 | }, 54 | "ignores undefined and NaN": function(reduce) { 55 | assert.equal(reduce([1, NaN, 3, undefined, null]), 3); 56 | }, 57 | "compares using natural order": function(reduce) { 58 | assert.equal(reduce([2, 10, 3]), 10); 59 | assert.equal(reduce(["2", "10", "3"]), "3"); 60 | assert.equal(reduce(["2", "10", 3]), 3); // "2" < "10", "10" < 3 61 | assert.equal(reduce([3, "2", "10"]), "10"); // "2" < 3, 3 < "10" 62 | }, 63 | "returns the first of equal values": function(reduce) { 64 | assert.strictEqual(reduce([1, new Number(1)]), 1); 65 | } 66 | }, 67 | 68 | "median": { 69 | topic: function() { 70 | return reduces.median; 71 | }, 72 | "empty is undefined": function(reduce) { 73 | assert.strictEqual(reduce.empty, undefined); 74 | }, 75 | "is not pyramidal": function(reduce) { 76 | assert.isTrue(!reduce.pyramidal); 77 | }, 78 | "returns the median value": function(reduce) { 79 | assert.equal(reduce([1, 2, 3, 2, 1]), 2); 80 | assert.equal(reduce([1, 2, 4, 2, 1, 4, 4, 4]), 3); 81 | }, 82 | "sorts input in-place": function(reduce) { 83 | var values = [1, 2, 3, 2, 1]; 84 | reduce(values); 85 | assert.deepEqual(values, [1, 1, 2, 2, 3]); 86 | }, 87 | "ignores undefined and NaN": function(reduce) { 88 | assert.equal(reduce([1, NaN, 3, undefined, 0]), 0); 89 | } 90 | }, 91 | 92 | "min": { 93 | topic: function() { 94 | return reduces.min; 95 | }, 96 | "empty is undefined": function(reduce) { 97 | assert.strictEqual(reduce.empty, undefined); 98 | }, 99 | "is pyramidal": function(reduce) { 100 | assert.isTrue(reduce.pyramidal); 101 | }, 102 | "returns the minimum value": function(reduce) { 103 | assert.equal(reduce([1, 2, 3, 2, 1]), 1); 104 | }, 105 | "ignores undefined and NaN": function(reduce) { 106 | assert.equal(reduce([1, NaN, 3, undefined, 0]), 0); 107 | }, 108 | "compares using natural order": function(reduce) { 109 | assert.equal(reduce([2, 10, 3]), 2); 110 | assert.equal(reduce(["2", "10", 3]), 3); // "2" > "10", 3 > "2" 111 | assert.equal(reduce([3, "2", "10"]), "10"); // 3 > "2", "2" > "10" 112 | }, 113 | "returns the first of equal values": function(reduce) { 114 | assert.strictEqual(reduce([1, new Number(1)]), 1); 115 | } 116 | }, 117 | 118 | "sum": { 119 | topic: function() { 120 | return reduces.sum; 121 | }, 122 | "empty is zero": function(reduce) { 123 | assert.strictEqual(reduce.empty, 0); 124 | }, 125 | "is pyramidal": function(reduce) { 126 | assert.isTrue(reduce.pyramidal); 127 | }, 128 | "returns the sum of values": function(reduce) { 129 | assert.equal(reduce([1, 2, 3, 2, 1]), 9); 130 | assert.equal(reduce([1, 2, 4, 2, 1, 4, 4, 4]), 22); 131 | }, 132 | "does not ignore undefined and NaN": function(reduce) { 133 | assert.isNaN(reduce([1, NaN, 3, undefined, 0])); 134 | } 135 | } 136 | }); 137 | 138 | suite.export(module); 139 | -------------------------------------------------------------------------------- /lib/cube/server.js: -------------------------------------------------------------------------------- 1 | var util = require("util"), 2 | url = require("url"), 3 | http = require("http"), 4 | dgram = require("dgram"), 5 | websocket = require("websocket"), 6 | websprocket = require("websocket-server"), 7 | static = require("node-static"), 8 | database = require('./database'); 9 | 10 | // And then this happened: 11 | websprocket.Connection = require("../../node_modules/websocket-server/lib/ws/connection"); 12 | 13 | // Configuration for WebSocket requests. 14 | var wsOptions = { 15 | maxReceivedFrameSize: 0x10000, 16 | maxReceivedMessageSize: 0x100000, 17 | fragmentOutgoingMessages: true, 18 | fragmentationThreshold: 0x4000, 19 | keepalive: true, 20 | keepaliveInterval: 20000, 21 | assembleFragments: true, 22 | disableNagleAlgorithm: true, 23 | closeTimeout: 5000 24 | }; 25 | 26 | module.exports = function(options) { 27 | 28 | // Don't crash on errors. 29 | process.on("uncaughtException", function(error) { 30 | util.log("uncaught exception: " + error); 31 | util.log(error.stack); 32 | }); 33 | 34 | var server = {}, 35 | primary = http.createServer(), 36 | secondary = websprocket.createServer(), 37 | file = new static.Server("static"), 38 | meta, 39 | endpoints = {ws: [], http: []}, 40 | id = 0; 41 | 42 | secondary.server = primary; 43 | 44 | // Register primary WebSocket listener with fallback. 45 | primary.on("upgrade", function(request, socket, head) { 46 | if ("sec-websocket-version" in request.headers) { 47 | request = new websocket.request(socket, request, wsOptions); 48 | request.readHandshake(); 49 | connect(request.accept(request.requestedProtocols[0], request.origin), request.httpRequest); 50 | } else if (request.method === "GET" 51 | && /^websocket$/i.test(request.headers.upgrade) 52 | && /^upgrade$/i.test(request.headers.connection)) { 53 | new websprocket.Connection(secondary.manager, secondary.options, request, socket, head); 54 | } 55 | }); 56 | 57 | // Register secondary WebSocket listener. 58 | secondary.on("connection", function(connection) { 59 | connection.socket = connection._socket; 60 | connection.remoteAddress = connection.socket.remoteAddress; 61 | connection.sendUTF = connection.send; 62 | connect(connection, connection._req); 63 | }); 64 | 65 | function connect(connection, request) { 66 | 67 | // Forward messages to the appropriate endpoint, or close the connection. 68 | for (var i = -1, n = endpoints.ws.length, e; ++i < n;) { 69 | if ((e = endpoints.ws[i]).match(request.url)) { 70 | 71 | var callback = function(response) { 72 | connection.sendUTF(JSON.stringify(response)); 73 | }; 74 | 75 | callback.id = ++id; 76 | 77 | // Listen for socket disconnect. 78 | if (e.dispatch.close) connection.socket.on("end", function() { 79 | e.dispatch.close(callback); 80 | }); 81 | 82 | connection.on("message", function(message) { 83 | e.dispatch(JSON.parse(message.utf8Data || message), callback); 84 | }); 85 | 86 | meta({ 87 | type: "cube_request", 88 | time: Date.now(), 89 | data: { 90 | ip: connection.remoteAddress, 91 | path: request.url, 92 | method: "WebSocket" 93 | } 94 | }); 95 | 96 | return; 97 | } 98 | } 99 | 100 | connection.close(); 101 | } 102 | 103 | // Register HTTP listener. 104 | primary.on("request", function(request, response) { 105 | var u = url.parse(request.url); 106 | 107 | // Forward messages to the appropriate endpoint, or 404. 108 | for (var i = -1, n = endpoints.http.length, e; ++i < n;) { 109 | if ((e = endpoints.http[i]).match(u.pathname, request.method)) { 110 | e.dispatch(request, response); 111 | 112 | meta({ 113 | type: "cube_request", 114 | time: Date.now(), 115 | data: { 116 | ip: request.connection.remoteAddress, 117 | path: u.pathname, 118 | method: request.method 119 | } 120 | }); 121 | 122 | return; 123 | } 124 | } 125 | 126 | // If this request wasn't matched, see if there's a static file to serve. 127 | request.on("end", function() { 128 | file.serve(request, response, function(error) { 129 | if (error) { 130 | response.writeHead(error.status, {"Content-Type": "text/plain"}); 131 | response.end(error.status + ""); 132 | } 133 | }); 134 | }); 135 | 136 | // as of node v0.10, 'end' is not emitted unless read() called 137 | if (request.read !== undefined) { 138 | request.read(); 139 | } 140 | }); 141 | 142 | server.start = function() { 143 | // Connect to mongodb. 144 | util.log("starting mongodb client"); 145 | database.open(options, function (error, db) { 146 | if (error) throw error; 147 | server.register(db, endpoints); 148 | meta = require("./event").putter(db); 149 | util.log("starting http server on port " + options["http-port"]); 150 | primary.listen(options["http-port"]); 151 | if (endpoints.udp) { 152 | util.log("starting udp server on port " + options["udp-port"]); 153 | var udp = dgram.createSocket("udp4"); 154 | udp.on("message", function(message) { 155 | endpoints.udp(JSON.parse(message.toString("utf8")), ignore); 156 | }); 157 | udp.bind(options["udp-port"]); 158 | } 159 | }); 160 | }; 161 | 162 | return server; 163 | }; 164 | 165 | function ignore() { 166 | // Responses for UDP are ignored; there's nowhere for them to go! 167 | } 168 | -------------------------------------------------------------------------------- /lib/cube/event-expression.peg: -------------------------------------------------------------------------------- 1 | // TODO allow null literals (requires fixing pegjs) 2 | 3 | { 4 | var filterEqual = function(o, k, v) { o[k] = v; }, 5 | filterGreater = filter("$gt"), 6 | filterGreaterOrEqual = filter("$gte"), 7 | filterLess = filter("$lt"), 8 | filterLessOrEqual = filter("$lte"), 9 | filterNotEqual = filter("$ne"), 10 | filterRegularExpression = filter("$regex"), 11 | filterIn = filter("$in"), 12 | exists = {$exists: true}; 13 | 14 | function noop() {} 15 | 16 | function filter(op) { 17 | return function(o, k, v) { 18 | var f = o[k]; 19 | switch (typeof f) { 20 | case "undefined": o[k] = f = {}; // continue 21 | case "object": f[op] = v; break; 22 | // otherwise, observe the existing equals (literal) filter 23 | } 24 | }; 25 | } 26 | 27 | function arrayAccessor(name) { 28 | name = new String(name); 29 | name.array = true; 30 | return name; 31 | } 32 | 33 | function objectAccessor(name) { 34 | return name; 35 | } 36 | 37 | function compoundFields(type, head, tail) { 38 | var n = tail.length; 39 | return { 40 | type: type, 41 | exists: function(o) { 42 | var i = -1; 43 | head.exists(o); 44 | while (++i < n) tail[i][3].exists(o); 45 | }, 46 | fields: function(o) { 47 | var i = -1; 48 | head.fields(o); 49 | while (++i < n) tail[i][3].fields(o); 50 | } 51 | }; 52 | } 53 | 54 | function member(head, tail) { 55 | var fields = ["d", head].concat(tail), 56 | shortName = fields.filter(function(d) { return !d.array; }).join("."), 57 | longName = fields.join("."), 58 | i = -1, 59 | n = fields.length; 60 | return { 61 | field: longName, 62 | exists: function(o) { 63 | if (!(shortName in o)) { 64 | o[shortName] = exists; 65 | } 66 | }, 67 | fields: function(o) { 68 | o[shortName] = 1; 69 | } 70 | }; 71 | } 72 | 73 | } 74 | 75 | start 76 | = _ expression:event_expression _ { expression.source = input; return expression; } 77 | 78 | event_expression 79 | = value:event_value_expression filters:(_ "." _ event_filter_expression)* 80 | { 81 | value.filter = function(filter) { 82 | var i = -1, n = filters.length; 83 | while (++i < n) filters[i][3](filter); 84 | value.exists(filter); 85 | }; 86 | return value; 87 | } 88 | 89 | event_filter_expression 90 | = op:filter_operator _ "(" _ member:event_member_expression _ "," _ value:literal _ ")" { return function(o) { op(o, member.field, value); }; } 91 | 92 | event_value_expression 93 | = type:type _ "(" _ head:event_member_expression tail:(_ "," _ event_member_expression)* _ ")" { return compoundFields(type, head, tail); } 94 | / type:type { return {type: type, exists: noop, fields: noop}; } 95 | 96 | event_member_expression 97 | = head:identifier tail:( 98 | _ "[" _ name:number _ "]" { return arrayAccessor(name); } 99 | / _ "." _ name:identifier { return objectAccessor(name); } 100 | )* { return member(head, tail); } 101 | 102 | filter_operator 103 | = "eq" { return filterEqual; } 104 | / "gt" { return filterGreater; } 105 | / "ge" { return filterGreaterOrEqual; } 106 | / "lt" { return filterLess; } 107 | / "le" { return filterLessOrEqual; } 108 | / "ne" { return filterNotEqual; } 109 | / "re" { return filterRegularExpression; } 110 | / "in" { return filterIn; } 111 | 112 | type 113 | = first:[a-z] rest:[a-zA-Z0-9_]+ { return first + rest.join(""); } 114 | 115 | identifier 116 | = first:[a-zA-Z_] rest:[a-zA-Z0-9_$]* { return first + rest.join(""); } 117 | 118 | literal 119 | = array_literal 120 | / string 121 | / number 122 | / "true" { return true; } 123 | / "false" { return false; } 124 | 125 | array_literal 126 | = "[" _ first:literal rest:(_ "," _ literal)* _ "]" { return [first].concat(rest.map(function(d) { return d[3]; })); } 127 | / "[" _ "]" { return []; } 128 | 129 | string "string" 130 | = '"' chars:double_string_char* '"' { return chars.join(""); } 131 | / "'" chars:single_string_char* "'" { return chars.join(""); } 132 | 133 | double_string_char 134 | = !('"' / "\\") char_:. { return char_; } 135 | / "\\" sequence:escape_sequence { return sequence; } 136 | 137 | single_string_char 138 | = !("'" / "\\") char_:. { return char_; } 139 | / "\\" sequence:escape_sequence { return sequence; } 140 | 141 | escape_sequence 142 | = character_escape_sequence 143 | / "0" !digit { return "\0"; } 144 | / hex_escape_sequence 145 | / unicode_escape_sequence 146 | 147 | character_escape_sequence 148 | = single_escape_character 149 | / non_escape_character 150 | 151 | single_escape_character 152 | = char_:['"\\bfnrtv] { return char_.replace("b", "\b").replace("f", "\f").replace("n", "\n").replace("r", "\r").replace("t", "\t").replace("v", "\x0B"); } 153 | 154 | non_escape_character 155 | = !escape_character char_:. { return char_; } 156 | 157 | escape_character 158 | = single_escape_character 159 | / digit 160 | / "x" 161 | / "u" 162 | 163 | hex_escape_sequence 164 | = "x" h1:hex_digit h2:hex_digit { return String.fromCharCode(+("0x" + h1 + h2)); } 165 | 166 | unicode_escape_sequence 167 | = "u" h1:hex_digit h2:hex_digit h3:hex_digit h4:hex_digit { return String.fromCharCode(+("0x" + h1 + h2 + h3 + h4)); } 168 | 169 | number "number" 170 | = "-" _ number:number { return -number; } 171 | / int_:int frac:frac exp:exp { return +(int_ + frac + exp); } 172 | / int_:int frac:frac { return +(int_ + frac); } 173 | / int_:int exp:exp { return +(int_ + exp); } 174 | / frac:frac { return +frac; } 175 | / int_:int { return +int_; } 176 | 177 | int 178 | = digit19:digit19 digits:digits { return digit19 + digits; } 179 | / digit:digit 180 | 181 | frac 182 | = "." digits:digits { return "." + digits; } 183 | 184 | exp 185 | = e:e digits:digits { return e + digits; } 186 | 187 | digits 188 | = digits:digit+ { return digits.join(""); } 189 | 190 | e 191 | = e:[eE] sign:[+-]? { return e + sign; } 192 | 193 | digit 194 | = [0-9] 195 | 196 | digit19 197 | = [1-9] 198 | 199 | hex_digit 200 | = [0-9a-fA-F] 201 | 202 | _ "whitespace" 203 | = whitespace* 204 | 205 | whitespace 206 | = [ \t\n\r] 207 | -------------------------------------------------------------------------------- /test/metric-test.js: -------------------------------------------------------------------------------- 1 | var vows = require("vows"), 2 | assert = require("assert"), 3 | test = require("./helpers"), 4 | event = require("../lib/cube/event"), 5 | metric = require("../lib/cube/metric"); 6 | 7 | var suite = vows.describe("metric"); 8 | 9 | var steps = { 10 | 1e4: function(date, n) { return new Date((Math.floor(date / 1e4) + n) * 1e4); }, 11 | 6e4: function(date, n) { return new Date((Math.floor(date / 6e4) + n) * 6e4); }, 12 | 3e5: function(date, n) { return new Date((Math.floor(date / 3e5) + n) * 3e5); }, 13 | 36e5: function(date, n) { return new Date((Math.floor(date / 36e5) + n) * 36e5); }, 14 | 864e5: function(date, n) { return new Date((Math.floor(date / 864e5) + n) * 864e5); } 15 | }; 16 | 17 | steps[1e4].description = "10-second"; 18 | steps[6e4].description = "1-minute"; 19 | steps[3e5].description = "5-minute"; 20 | steps[36e5].description = "1-hour"; 21 | steps[864e5].description = "1-day"; 22 | 23 | suite.addBatch(test.batch({ 24 | topic: function(test) { 25 | var putter = event.putter(test.db), 26 | getter = metric.getter(test.db), 27 | callback = this.callback; 28 | 29 | for (var i = 0; i < 2500; i++) { 30 | putter({ 31 | type: "test", 32 | time: new Date(Date.UTC(2011, 6, 18, 0, Math.sqrt(i) - 10)).toISOString(), 33 | data: {i: i} 34 | }); 35 | } 36 | 37 | function waitForEvents() { 38 | test.db.collection("test_events").count(function(err,count) { 39 | if (count == 2500) { 40 | callback(null, getter); 41 | } else { 42 | setTimeout(waitForEvents, 10); 43 | } 44 | }); 45 | } 46 | 47 | setTimeout(waitForEvents,10); 48 | }, 49 | 50 | "unary expression": metricTest({ 51 | expression: "sum(test)", 52 | start: "2011-07-17T23:47:00.000Z", 53 | stop: "2011-07-18T00:50:00.000Z", 54 | }, { 55 | 6e4: [0, 0, 0, 1, 1, 3, 5, 7, 9, 11, 13, 15, 17, 39, 23, 25, 27, 29, 31, 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63, 65, 67, 69, 71, 73, 75, 77, 79, 81, 83, 85, 87, 89, 91, 93, 95, 97, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 56 | 3e5: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], 57 | 36e5: [82, 2418], 58 | 864e5: [82, 2418] 59 | } 60 | ), 61 | 62 | "unary expression with data accessor": metricTest({ 63 | expression: "sum(test(i))", 64 | start: "2011-07-17T23:47:00.000Z", 65 | stop: "2011-07-18T00:50:00.000Z" 66 | }, { 67 | 3e5: [0, 136, 3185, 21879, 54600, 115200, 209550, 345150, 529500, 770100, 1074450, 0, 0], 68 | 36e5: [3321, 3120429], 69 | 864e5: [3321, 3120429] 70 | } 71 | ), 72 | 73 | "unary expression with compound data accessor": metricTest({ 74 | expression: "sum(test(i / 100))", 75 | start: "2011-07-17T23:47:00.000Z", 76 | stop: "2011-07-18T00:50:00.000Z" 77 | }, { 78 | 3e5: [0, 1.36, 31.85, 218.79, 546, 1152, 2095.5, 3451.5, 5295, 7701, 10744.5, 0, 0], 79 | 36e5: [33.21, 31204.29], 80 | 864e5: [33.21, 31204.29] 81 | } 82 | ), 83 | 84 | "max expression": metricTest({ 85 | expression: "max(test(i))", 86 | start: "2011-07-17T23:47:00.000Z", 87 | stop: "2011-07-18T00:50:00.000Z", 88 | }, { 89 | 36e5: [81, 2499], 90 | 864e5: [81, 2499] 91 | } 92 | ), 93 | 94 | "min expression": metricTest({ 95 | expression: "min(test(i))", 96 | start: "2011-07-17T23:47:00.000Z", 97 | stop: "2011-07-18T00:50:00.000Z", 98 | }, { 99 | 36e5: [0, 82], 100 | 864e5: [0, 82] 101 | } 102 | ), 103 | 104 | "compound expression": metricTest({ 105 | expression: "max(test(i)) - min(test(i))", 106 | start: "2011-07-17T23:47:00.000Z", 107 | stop: "2011-07-18T00:50:00.000Z", 108 | }, { 109 | 3e5: [NaN, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, NaN, NaN], 110 | 36e5: [81, 2417], 111 | 864e5: [81, 2417] 112 | } 113 | ), 114 | 115 | "non-pyramidal expression": metricTest({ 116 | expression: "distinct(test(i))", 117 | start: "2011-07-17T23:47:00.000Z", 118 | stop: "2011-07-18T00:50:00.000Z", 119 | }, { 120 | 3e5: [0, 17, 65, 143, 175, 225, 275, 325, 375, 425, 475, 0, 0], 121 | 36e5: [82, 2418], 122 | 864e5: [82, 2418] 123 | } 124 | ), 125 | 126 | "compound pyramidal and non-pyramidal expression": metricTest({ 127 | expression: "sum(test(i)) - median(test(i))", 128 | start: "2011-07-17T23:47:00.000Z", 129 | stop: "2011-07-18T00:50:00.000Z", 130 | }, { 131 | 3e5: [NaN, 128, 3136, 21726, 54288, 114688, 208788, 344088, 528088, 768288, 1072188, NaN, NaN], 132 | 36e5: [3280.5, 3119138.5], 133 | 864e5: [3280.5, 3119138.5] 134 | } 135 | ), 136 | 137 | "compound with constant expression": metricTest({ 138 | expression: "-1 + sum(test)", 139 | start: "2011-07-17T23:47:00.000Z", 140 | stop: "2011-07-18T00:50:00.000Z", 141 | }, { 142 | 3e5: [-1, 16, 64, 142, 174, 224, 274, 324, 374, 424, 474, -1, -1], 143 | 36e5: [81, 2417], 144 | 864e5: [81, 2417] 145 | } 146 | ) 147 | })); 148 | 149 | suite.export(module); 150 | 151 | function metricTest(request, expected) { 152 | var t = {}, k; 153 | for (k in expected) t["at " + steps[k].description + " intervals"] = testStep(k, expected[k]); 154 | return t; 155 | 156 | function testStep(step, expected) { 157 | var t = testStepDepth(0, step, expected); 158 | t["(cached)"] = testStepDepth(1, step, expected); 159 | return t; 160 | } 161 | 162 | function testStepDepth(depth, step, expected) { 163 | var start = new Date(request.start), 164 | stop = new Date(request.stop); 165 | 166 | var test = { 167 | topic: function() { 168 | var actual = [], 169 | timeout = setTimeout(function() { cb("Time's up!"); }, 10000), 170 | cb = this.callback, 171 | req = Object.create(request), 172 | test = arguments[depth]; 173 | req.step = step; 174 | setTimeout(function() { 175 | test(req, function(response) { 176 | if (response.time >= stop) { 177 | clearTimeout(timeout); 178 | cb(null, actual.sort(function(a, b) { return a.time - b.time; })); 179 | } else { 180 | actual.push(response); 181 | } 182 | }); 183 | }, depth * 250); 184 | } 185 | }; 186 | 187 | test[request.expression] = function(actual) { 188 | 189 | // rounds down the start time (inclusive) 190 | var floor = steps[step](start, 0); 191 | assert.deepEqual(actual[0].time, floor); 192 | 193 | // rounds up the stop time (exclusive) 194 | var ceil = steps[step](stop, 0); 195 | if (!(ceil - stop)) ceil = steps[step](stop, -1); 196 | assert.deepEqual(actual[actual.length - 1].time, ceil); 197 | 198 | // formats UTC time in ISO 8601 199 | actual.forEach(function(d) { 200 | assert.instanceOf(d.time, Date); 201 | assert.match(JSON.stringify(d.time), /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:00.000Z/); 202 | }); 203 | 204 | // returns exactly one value per time 205 | var i = 0, n = actual.length, t = actual[0].time; 206 | while (++i < n) assert.isTrue(t < (t = actual[i].time)); 207 | 208 | // each metric defines only time and value properties 209 | actual.forEach(function(d) { 210 | assert.deepEqual(Object.keys(d), ["time", "value"]); 211 | }); 212 | 213 | // returns the expected times 214 | var floor = steps[step], 215 | time = floor(start, 0), 216 | times = []; 217 | while (time < stop) { 218 | times.push(time); 219 | time = floor(time, 1); 220 | } 221 | assert.deepEqual(actual.map(function(d) { return d.time; }), times); 222 | 223 | // returns the expected values 224 | var actualValues = actual.map(function(d) { return d.value; }); 225 | assert.equal(expected.length, actual.length, "expected " + expected + ", got " + actualValues); 226 | expected.forEach(function(value, i) { 227 | if (Math.abs(actual[i].value - value) > 1e-6) { 228 | assert.fail(actual.map(function(d) { return d.value; }), expected, "expected {expected}, got {actual} at " + actual[i].time.toISOString()); 229 | } 230 | }); 231 | 232 | }; 233 | 234 | return test; 235 | } 236 | } 237 | -------------------------------------------------------------------------------- /test/event-expression-test.js: -------------------------------------------------------------------------------- 1 | var vows = require("vows"), 2 | assert = require("assert"), 3 | parser = require("../lib/cube/event-expression"); 4 | 5 | var suite = vows.describe("event-expression"); 6 | 7 | suite.addBatch({ 8 | 9 | "a simple event expression, test": { 10 | topic: parser.parse("test"), 11 | "has the expected event type, test": function(e) { 12 | assert.equal(e.type, "test"); 13 | }, 14 | "does not load any event data fields": function(e) { 15 | var fields = {}; 16 | e.fields(fields); 17 | assert.deepEqual(fields, {}); 18 | }, 19 | "does not apply any event data filters": function(e) { 20 | var filter = {}; 21 | e.filter(filter); 22 | assert.deepEqual(filter, {}); 23 | }, 24 | "has the expected source": function(e) { 25 | assert.equal(e.source, "test"); 26 | } 27 | }, 28 | 29 | "an expression with a single field": { 30 | topic: parser.parse("test(i)"), 31 | "loads the specified event data field": function(e) { 32 | var fields = {}; 33 | e.fields(fields); 34 | assert.deepEqual(fields, {"d.i": 1}); 35 | }, 36 | "ignores events that do not have the specified field": function(e) { 37 | var filter = {}; 38 | e.filter(filter); 39 | assert.deepEqual(filter, {"d.i": {$exists: true}}); 40 | }, 41 | "has the expected source": function(e) { 42 | assert.equal(e.source, "test(i)"); 43 | } 44 | }, 45 | 46 | "an expression with multiple fields": { 47 | topic: parser.parse("test(i, j)"), 48 | "loads the specified event data fields": function(e) { 49 | var fields = {}; 50 | e.fields(fields); 51 | assert.deepEqual(fields, {"d.i": 1, "d.j": 1}); 52 | }, 53 | "ignores events that do not have the specified fields": function(e) { 54 | var filter = {}; 55 | e.filter(filter); 56 | assert.deepEqual(filter, {"d.i": {$exists: true}, "d.j": {$exists: true}}); 57 | }, 58 | "has the expected source": function(e) { 59 | assert.equal(e.source, "test(i, j)"); 60 | } 61 | }, 62 | 63 | "an expression with a filter on the requested field": { 64 | topic: parser.parse("test(i).gt(i, 42)"), 65 | "loads the specified event data fields": function(e) { 66 | var fields = {}; 67 | e.fields(fields); 68 | assert.deepEqual(fields, {"d.i": 1}); 69 | }, 70 | "only filters using the explicit filter; existence is implied": function(e) { 71 | var filter = {}; 72 | e.filter(filter); 73 | assert.deepEqual(filter, {"d.i": {$gt: 42}}); 74 | }, 75 | "has the expected source": function(e) { 76 | assert.equal(e.source, "test(i).gt(i, 42)"); 77 | } 78 | }, 79 | 80 | "an expression with filters on different fields": { 81 | topic: parser.parse("test.gt(i, 42).eq(j, \"foo\")"), 82 | "does not load fields that are only filtered": function(e) { 83 | var fields = {}; 84 | e.fields(fields); 85 | assert.deepEqual(fields, {}); 86 | }, 87 | "has the expected filters on each specified field": function(e) { 88 | var filter = {}; 89 | e.filter(filter); 90 | assert.deepEqual(filter, {"d.i": {$gt: 42}, "d.j": "foo"}); 91 | } 92 | }, 93 | 94 | "an expression with multiple filters on the same field": { 95 | topic: parser.parse("test.gt(i, 42).le(i, 52)"), 96 | "combines multiple filters on the specified field": function(e) { 97 | var filter = {}; 98 | e.filter(filter); 99 | assert.deepEqual(filter, {"d.i": {$gt: 42, $lte: 52}}); 100 | }, 101 | "has the expected source": function(e) { 102 | assert.equal(e.source, "test.gt(i, 42).le(i, 52)"); 103 | } 104 | }, 105 | 106 | "an expression with range and exact filters on the same field": { 107 | topic: parser.parse("test.gt(i, 42).eq(i, 52)"), 108 | "ignores range filters, taking only the exact filter": function(e) { 109 | var filter = {}; 110 | e.filter(filter); 111 | assert.deepEqual(filter, {"d.i": 52}); 112 | }, 113 | "has the expected source": function(e) { 114 | assert.equal(e.source, "test.gt(i, 42).eq(i, 52)"); 115 | } 116 | }, 117 | 118 | "an expression with exact and range filters on the same field": { 119 | topic: parser.parse("test.eq(i, 52).gt(i, 42)"), 120 | "ignores range filters, taking only the exact filter": function(e) { 121 | var filter = {}; 122 | e.filter(filter); 123 | assert.deepEqual(filter, {"d.i": 52}); 124 | }, 125 | "has the expected source": function(e) { 126 | assert.equal(e.source, "test.eq(i, 52).gt(i, 42)"); 127 | } 128 | }, 129 | 130 | "an expression with an object data accessor": { 131 | topic: parser.parse("test(i.j)"), 132 | "loads the specified event data field": function(e) { 133 | var fields = {}; 134 | e.fields(fields); 135 | assert.deepEqual(fields, {"d.i.j": 1}); 136 | }, 137 | "ignores events that do not have the specified field": function(e) { 138 | var filter = {}; 139 | e.filter(filter); 140 | assert.deepEqual(filter, {"d.i.j": {$exists: true}}); 141 | }, 142 | "has the expected source": function(e) { 143 | assert.equal(e.source, "test(i.j)"); 144 | } 145 | }, 146 | 147 | "an expression with an array data accessor": { 148 | topic: parser.parse("test(i[0])"), 149 | "loads the specified event data field": function(e) { 150 | var fields = {}; 151 | e.fields(fields); 152 | assert.deepEqual(fields, {"d.i": 1}); 153 | }, 154 | "ignores events that do not have the specified field": function(e) { 155 | var filter = {}; 156 | e.filter(filter); 157 | assert.deepEqual(filter, {"d.i": {$exists: true}}); 158 | }, 159 | "has the expected source": function(e) { 160 | assert.equal(e.source, "test(i[0])"); 161 | } 162 | }, 163 | 164 | "an expression with an elaborate data accessor": { 165 | topic: parser.parse("test(i.j[0].k)"), 166 | "loads the specified event data field": function(e) { 167 | var fields = {}; 168 | e.fields(fields); 169 | assert.deepEqual(fields, {"d.i.j.k": 1}); 170 | }, 171 | "ignores events that do not have the specified field": function(e) { 172 | var filter = {}; 173 | e.filter(filter); 174 | assert.deepEqual(filter, {"d.i.j.k": {$exists: true}}); 175 | }, 176 | "has the expected source": function(e) { 177 | assert.equal(e.source, "test(i.j[0].k)"); 178 | } 179 | }, 180 | 181 | "an expression with an elaborate filter": { 182 | topic: parser.parse("test.gt(i.j[0].k, 42)"), 183 | "does not load fields that are only filtered": function(e) { 184 | var fields = {}; 185 | e.fields(fields); 186 | assert.deepEqual(fields, {}); 187 | }, 188 | "has the expected filter": function(e) { 189 | var filter = {}; 190 | e.filter(filter); 191 | assert.deepEqual(filter, {"d.i.j.0.k": {$gt: 42}}); 192 | }, 193 | "has the expected source": function(e) { 194 | assert.equal(e.source, "test.gt(i.j[0].k, 42)"); 195 | } 196 | }, 197 | 198 | "filters": { 199 | "the eq filter results in a simple query filter": function() { 200 | var filter = {}; 201 | parser.parse("test.eq(i, 42)").filter(filter); 202 | assert.deepEqual(filter, {"d.i": 42}); 203 | }, 204 | "the gt filter results in a $gt query filter": function(e) { 205 | var filter = {}; 206 | parser.parse("test.gt(i, 42)").filter(filter); 207 | assert.deepEqual(filter, {"d.i": {$gt: 42}}); 208 | }, 209 | "the ge filter results in a $gte query filter": function(e) { 210 | var filter = {}; 211 | parser.parse("test.ge(i, 42)").filter(filter); 212 | assert.deepEqual(filter, {"d.i": {$gte: 42}}); 213 | }, 214 | "the lt filter results in an $lt query filter": function(e) { 215 | var filter = {}; 216 | parser.parse("test.lt(i, 42)").filter(filter); 217 | assert.deepEqual(filter, {"d.i": {$lt: 42}}); 218 | }, 219 | "the le filter results in an $lte query filter": function(e) { 220 | var filter = {}; 221 | parser.parse("test.le(i, 42)").filter(filter); 222 | assert.deepEqual(filter, {"d.i": {$lte: 42}}); 223 | }, 224 | "the ne filter results in an $ne query filter": function(e) { 225 | var filter = {}; 226 | parser.parse("test.ne(i, 42)").filter(filter); 227 | assert.deepEqual(filter, {"d.i": {$ne: 42}}); 228 | }, 229 | "the re filter results in a $regex query filter": function(e) { 230 | var filter = {}; 231 | parser.parse("test.re(i, \"foo\")").filter(filter); 232 | assert.deepEqual(filter, {"d.i": {$regex: "foo"}}); 233 | }, 234 | "the in filter results in a $in query filter": function(e) { 235 | var filter = {}; 236 | parser.parse("test.in(i, [\"foo\", 42])").filter(filter); 237 | assert.deepEqual(filter, {"d.i": {$in: ["foo", 42]}}); 238 | } 239 | } 240 | 241 | }); 242 | 243 | suite.export(module); 244 | -------------------------------------------------------------------------------- /lib/cube/metric.js: -------------------------------------------------------------------------------- 1 | // TODO use expression ids or hashes for more compact storage 2 | 3 | var parser = require("./metric-expression"), 4 | tiers = require("./tiers"), 5 | types = require("./types"), 6 | reduces = require("./reduces"), 7 | event = require("./event"), 8 | setImmediate = require("./set-immediate"); 9 | 10 | var metric_fields = {v: 1}, 11 | metric_options = {sort: {"_id.t": 1}, batchSize: 1000}, 12 | event_options = {sort: {t: 1}, batchSize: 1000}; 13 | 14 | // Query for metrics. 15 | exports.getter = function(db) { 16 | var collection = types(db), 17 | Double = db.bson_serializer.Double, 18 | queueByName = {}, 19 | meta = event.putter(db); 20 | 21 | function getter(request, callback) { 22 | var start = new Date(request.start), 23 | stop = new Date(request.stop), 24 | id = request.id; 25 | 26 | // Validate the dates. 27 | if (isNaN(start)) return callback({error: "invalid start"}), -1; 28 | if (isNaN(stop)) return callback({error: "invalid stop"}), -1; 29 | 30 | // Parse the expression. 31 | var expression; 32 | try { 33 | expression = parser.parse(request.expression); 34 | } catch (e) { 35 | return callback({error: "invalid expression"}), -1; 36 | } 37 | 38 | // Round start and stop to the appropriate time step. 39 | var tier = tiers[+request.step]; 40 | if (!tier) return callback({error: "invalid step"}), -1; 41 | start = tier.floor(start); 42 | stop = tier.ceil(stop); 43 | 44 | // Compute the request metric! 45 | measure(expression, start, stop, tier, "id" in request 46 | ? function(time, value) { callback({time: time, value: value, id: id}); } 47 | : function(time, value) { callback({time: time, value: value}); }); 48 | } 49 | 50 | // Computes the metric for the given expression for the time interval from 51 | // start (inclusive) to stop (exclusive). The time granularity is determined 52 | // by the specified tier, such as daily or hourly. The callback is invoked 53 | // repeatedly for each metric value, being passed two arguments: the time and 54 | // the value. The values may be out of order due to partial cache hits. 55 | function measure(expression, start, stop, tier, callback) { 56 | (expression.op ? binary : expression.type ? unary : constant)(expression, start, stop, tier, callback); 57 | } 58 | 59 | // Computes a constant expression; 60 | function constant(expression, start, stop, tier, callback) { 61 | var value = expression.value(); 62 | while (start < stop) { 63 | callback(start, value); 64 | start = tier.step(start); 65 | } 66 | callback(stop); 67 | } 68 | 69 | // Serializes a unary expression for computation. 70 | function unary(expression, start, stop, tier, callback) { 71 | var remaining = 0, 72 | time0 = Date.now(), 73 | time = start, 74 | name = expression.source, 75 | queue = queueByName[name], 76 | step = tier.key; 77 | 78 | // Compute the expected number of values. 79 | while (time < stop) ++remaining, time = tier.step(time); 80 | 81 | // If no results were requested, return immediately. 82 | if (!remaining) return callback(stop); 83 | 84 | // Add this task to the appropriate queue. 85 | if (queue) queue.next = task; 86 | else setImmediate(task); 87 | queueByName[name] = task; 88 | 89 | function task() { 90 | findOrComputeUnary(expression, start, stop, tier, function(time, value) { 91 | callback(time, value); 92 | if (!--remaining) { 93 | callback(stop); 94 | if (task.next) setImmediate(task.next); 95 | else delete queueByName[name]; 96 | 97 | // Record how long it took us to compute as an event! 98 | var time1 = Date.now(); 99 | meta({ 100 | type: "cube_compute", 101 | time: time1, 102 | data: { 103 | expression: expression.source, 104 | ms: time1 - time0 105 | } 106 | }); 107 | } 108 | }); 109 | } 110 | } 111 | 112 | // Finds or computes a unary (primary) expression. 113 | function findOrComputeUnary(expression, start, stop, tier, callback) { 114 | var name = expression.type, 115 | type = collection(name), 116 | map = expression.value, 117 | reduce = reduces[expression.reduce], 118 | filter = {t: {}}, 119 | fields = {t: 1}; 120 | 121 | // Copy any expression filters into the query object. 122 | expression.filter(filter); 123 | 124 | // Request any needed fields. 125 | expression.fields(fields); 126 | 127 | find(start, stop, tier, callback); 128 | 129 | // The metric is computed recursively, reusing the above variables. 130 | function find(start, stop, tier, callback) { 131 | var compute = tier.next && reduce.pyramidal ? computePyramidal : computeFlat, 132 | step = tier.key; 133 | 134 | // Query for the desired metric in the cache. 135 | type.metrics.find({ 136 | i: false, 137 | "_id.e": expression.source, 138 | "_id.l": tier.key, 139 | "_id.t": { 140 | $gte: start, 141 | $lt: stop 142 | } 143 | }, metric_fields, metric_options, foundMetrics); 144 | 145 | // Immediately report back whatever we have. If any values are missing, 146 | // merge them into contiguous intervals and asynchronously compute them. 147 | function foundMetrics(error, cursor) { 148 | handle(error); 149 | var time = start; 150 | cursor.each(function(error, row) { 151 | handle(error); 152 | if (row) { 153 | callback(row._id.t, row.v); 154 | if (time < row._id.t) compute(time, row._id.t); 155 | time = tier.step(row._id.t); 156 | } else { 157 | if (time < stop) compute(time, stop); 158 | } 159 | }); 160 | } 161 | 162 | // Group metrics from the next tier. 163 | function computePyramidal(start, stop) { 164 | var bins = {}; 165 | find(start, stop, tier.next, function(time, value) { 166 | var bin = bins[time = tier.floor(time)] || (bins[time] = {size: tier.size(time), values: []}); 167 | if (bin.values.push(value) === bin.size) { 168 | save(time, reduce(bin.values)); 169 | delete bins[time]; 170 | } 171 | }); 172 | } 173 | 174 | // Group raw events. Unlike the pyramidal computation, here we can control 175 | // the order in which rows are returned from the database. Thus, we know 176 | // when we've seen all of the events for a given time interval. 177 | function computeFlat(start, stop) { 178 | filter.t.$gte = start; 179 | filter.t.$lt = stop; 180 | type.events.find(filter, fields, event_options, function(error, cursor) { 181 | handle(error); 182 | var time = start, values = []; 183 | cursor.each(function(error, row) { 184 | handle(error); 185 | if (row) { 186 | var then = tier.floor(row.t); 187 | if (time < then) { 188 | save(time, values.length ? reduce(values) : reduce.empty); 189 | while ((time = tier.step(time)) < then) save(time, reduce.empty); 190 | values = [map(row)]; 191 | } else { 192 | values.push(map(row)); 193 | } 194 | } else { 195 | save(time, values.length ? reduce(values) : reduce.empty); 196 | while ((time = tier.step(time)) < stop) save(time, reduce.empty); 197 | } 198 | }); 199 | }); 200 | } 201 | 202 | function save(time, value) { 203 | callback(time, value); 204 | if (value) { 205 | type.metrics.save({ 206 | _id: { 207 | e: expression.source, 208 | l: tier.key, 209 | t: time 210 | }, 211 | i: false, 212 | v: new Double(value) 213 | }, handle); 214 | } 215 | } 216 | } 217 | } 218 | 219 | // Computes a binary expression by merging two subexpressions. 220 | function binary(expression, start, stop, tier, callback) { 221 | var left = {}, right = {}; 222 | 223 | measure(expression.left, start, stop, tier, function(t, l) { 224 | if (t in right) { 225 | callback(t, t < stop ? expression.op(l, right[t]) : l); 226 | delete right[t]; 227 | } else { 228 | left[t] = l; 229 | } 230 | }); 231 | 232 | measure(expression.right, start, stop, tier, function(t, r) { 233 | if (t in left) { 234 | callback(t, t < stop ? expression.op(left[t], r) : r); 235 | delete left[t]; 236 | } else { 237 | right[t] = r; 238 | } 239 | }); 240 | } 241 | 242 | return getter; 243 | }; 244 | 245 | function handle(error) { 246 | if (error) throw error; 247 | } 248 | -------------------------------------------------------------------------------- /lib/cube/metric-expression.peg: -------------------------------------------------------------------------------- 1 | // TODO allow null literals (requires fixing pegjs) 2 | 3 | { 4 | var filterEqual = function(o, k, v) { o[k] = v; }, 5 | filterGreater = filter("$gt"), 6 | filterGreaterOrEqual = filter("$gte"), 7 | filterLess = filter("$lt"), 8 | filterLessOrEqual = filter("$lte"), 9 | filterNotEqual = filter("$ne"), 10 | filterRegularExpression = filter("$regex"), 11 | filterIn = filter("$in"), 12 | exists = {$exists: true}; 13 | 14 | function add(a, b) { return a + b; } 15 | function subtract(a, b) { return a - b; } 16 | function multiply(a, b) { return a * b; } 17 | function divide(a, b) { return a / b; } 18 | 19 | function one() { return 1; } 20 | function noop() {} 21 | 22 | function filter(op) { 23 | return function(o, k, v) { 24 | var f = o[k]; 25 | switch (typeof f) { 26 | case "undefined": o[k] = f = {}; // continue 27 | case "object": f[op] = v; break; 28 | // otherwise, observe the existing equals (literal) filter 29 | } 30 | }; 31 | } 32 | 33 | function arrayAccessor(name) { 34 | name = new String(name); 35 | name.array = true; 36 | return name; 37 | } 38 | 39 | function objectAccessor(name) { 40 | return name; 41 | } 42 | 43 | function compoundMetric(head, tail) { 44 | var i = -1, 45 | n = tail.length, 46 | t, 47 | e = head; 48 | while (++i < n) { 49 | t = tail[i]; 50 | e = {left: e, op: t[1], right: t[3]}; 51 | if (!i) head = e; 52 | } 53 | return head; 54 | } 55 | 56 | function compoundValue(head, tail) { 57 | var n = tail.length; 58 | return { 59 | exists: function(o) { 60 | var i = -1; 61 | head.exists(o); 62 | while (++i < n) tail[i][3].exists(o); 63 | }, 64 | fields: function(o) { 65 | var i = -1; 66 | head.fields(o); 67 | while (++i < n) tail[i][3].fields(o); 68 | }, 69 | value: function(o) { 70 | var v = head.value(o), 71 | i = -1, 72 | t; 73 | while (++i < n) v = (t = tail[i])[1](v, t[3].value(o)); 74 | return v; 75 | } 76 | }; 77 | } 78 | 79 | function member(head, tail) { 80 | var fields = ["d", head].concat(tail), 81 | shortName = fields.filter(function(d) { return !d.array; }).join("."), 82 | longName = fields.join("."), 83 | i = -1, 84 | n = fields.length; 85 | return { 86 | field: longName, 87 | exists: function(o) { 88 | if (!(shortName in o)) { 89 | o[shortName] = exists; 90 | } 91 | }, 92 | fields: function(o) { 93 | o[shortName] = 1; 94 | }, 95 | value: function(o) { 96 | var i = -1; 97 | while (++i < n) { 98 | o = o[fields[i]]; 99 | } 100 | return o; 101 | } 102 | }; 103 | } 104 | } 105 | 106 | start 107 | = _ expression:metric_additive_expression _ { return expression; } 108 | 109 | metric_additive_expression 110 | = head:metric_multiplicative_expression tail:(_ additive_operator _ metric_additive_expression)* { return compoundMetric(head, tail); } 111 | 112 | metric_multiplicative_expression 113 | = head:metric_unary_expression tail:(_ multiplicative_operator _ metric_multiplicative_expression)* { return compoundMetric(head, tail); } 114 | 115 | metric_unary_expression 116 | = "-" _ expression:metric_unary_expression { var value = expression.value; expression.value = function(o) { return -value(o); }; if (expression.source) expression.source = "-" + expression.source; return expression; } 117 | / metric_primary_expression 118 | 119 | metric_primary_expression 120 | = reduce:reduce _ "(" _ event:event_expression _ ")" { event.reduce = reduce; event.source = input.substring(savedPos3, pos); return event; } 121 | / value:number { return {value: function() { return value; }}; } 122 | / "(" _ expression:metric_additive_expression _ ")" { return expression; } 123 | 124 | event_expression 125 | = value:event_value_expression filters:(_ "." _ event_filter_expression)* 126 | { 127 | value.filter = function(filter) { 128 | var i = -1, n = filters.length; 129 | while (++i < n) filters[i][3](filter); 130 | value.exists(filter); 131 | }; 132 | return value; 133 | } 134 | 135 | event_filter_expression 136 | = op:filter_operator _ "(" _ member:event_member_expression _ "," _ value:literal _ ")" { return function(o) { op(o, member.field, value); }; } 137 | 138 | event_value_expression 139 | = type:type _ "(" _ value:event_additive_expression _ ")" { value.type = type; return value; } 140 | / type:type { return {type: type, value: one, exists: noop, fields: noop}; } 141 | 142 | event_additive_expression 143 | = head:event_multiplicative_expression tail:(_ additive_operator _ event_additive_expression)* { return compoundValue(head, tail); } 144 | 145 | event_multiplicative_expression 146 | = head:event_unary_expression tail:(_ multiplicative_operator _ event_multiplicative_expression)* { return compoundValue(head, tail); } 147 | 148 | event_unary_expression 149 | = event_primary_expression 150 | / "-" _ unary:event_unary_expression { return {value: function(o) { return -unary.value(o); }, exists: unary.exists, fields: unary.fields}; } 151 | 152 | event_primary_expression 153 | = event_member_expression 154 | / number:number { return {value: function() { return number; }, exists: noop, fields: noop}; } 155 | / "(" _ expression:event_additive_expression _ ")" { return expression; } 156 | 157 | event_member_expression 158 | = head:identifier tail:( 159 | _ "[" _ name:number _ "]" { return arrayAccessor(name); } 160 | / _ "." _ name:identifier { return objectAccessor(name); } 161 | )* { return member(head, tail); } 162 | 163 | additive_operator 164 | = "+" { return add; } 165 | / "-" { return subtract; } 166 | 167 | multiplicative_operator 168 | = "*" { return multiply; } 169 | / "/" { return divide; } 170 | 171 | filter_operator 172 | = "eq" { return filterEqual; } 173 | / "gt" { return filterGreater; } 174 | / "ge" { return filterGreaterOrEqual; } 175 | / "lt" { return filterLess; } 176 | / "le" { return filterLessOrEqual; } 177 | / "ne" { return filterNotEqual; } 178 | / "re" { return filterRegularExpression; } 179 | / "in" { return filterIn; } 180 | 181 | reduce 182 | = "sum" 183 | / "min" 184 | / "max" 185 | / "distinct" 186 | / "median" 187 | 188 | type 189 | = first:[a-z] rest:[a-zA-Z0-9_]+ { return first + rest.join(""); } 190 | 191 | identifier 192 | = first:[a-zA-Z_] rest:[a-zA-Z0-9_$]* { return first + rest.join(""); } 193 | 194 | literal 195 | = array_literal 196 | / string 197 | / number 198 | / "true" { return true; } 199 | / "false" { return false; } 200 | 201 | array_literal 202 | = "[" _ first:literal rest:(_ "," _ literal)* _ "]" { return [first].concat(rest.map(function(d) { return d[3]; })); } 203 | / "[" _ "]" { return []; } 204 | 205 | string "string" 206 | = '"' chars:double_string_char* '"' { return chars.join(""); } 207 | / "'" chars:single_string_char* "'" { return chars.join(""); } 208 | 209 | double_string_char 210 | = !('"' / "\\") char_:. { return char_; } 211 | / "\\" sequence:escape_sequence { return sequence; } 212 | 213 | single_string_char 214 | = !("'" / "\\") char_:. { return char_; } 215 | / "\\" sequence:escape_sequence { return sequence; } 216 | 217 | escape_sequence 218 | = character_escape_sequence 219 | / "0" !digit { return "\0"; } 220 | / hex_escape_sequence 221 | / unicode_escape_sequence 222 | 223 | character_escape_sequence 224 | = single_escape_character 225 | / non_escape_character 226 | 227 | single_escape_character 228 | = char_:['"\\bfnrtv] { return char_.replace("b", "\b").replace("f", "\f").replace("n", "\n").replace("r", "\r").replace("t", "\t").replace("v", "\x0B"); } 229 | 230 | non_escape_character 231 | = !escape_character char_:. { return char_; } 232 | 233 | escape_character 234 | = single_escape_character 235 | / digit 236 | / "x" 237 | / "u" 238 | 239 | hex_escape_sequence 240 | = "x" h1:hex_digit h2:hex_digit { return String.fromCharCode(+("0x" + h1 + h2)); } 241 | 242 | unicode_escape_sequence 243 | = "u" h1:hex_digit h2:hex_digit h3:hex_digit h4:hex_digit { return String.fromCharCode(+("0x" + h1 + h2 + h3 + h4)); } 244 | 245 | number "number" 246 | = "-" _ number:number { return -number; } 247 | / int_:int frac:frac exp:exp { return +(int_ + frac + exp); } 248 | / int_:int frac:frac { return +(int_ + frac); } 249 | / int_:int exp:exp { return +(int_ + exp); } 250 | / frac:frac { return +frac; } 251 | / int_:int { return +int_; } 252 | 253 | int 254 | = digit19:digit19 digits:digits { return digit19 + digits; } 255 | / digit:digit 256 | 257 | frac 258 | = "." digits:digits { return "." + digits; } 259 | 260 | exp 261 | = e:e digits:digits { return e + digits; } 262 | 263 | digits 264 | = digits:digit+ { return digits.join(""); } 265 | 266 | e 267 | = e:[eE] sign:[+-]? { return e + sign; } 268 | 269 | digit 270 | = [0-9] 271 | 272 | digit19 273 | = [1-9] 274 | 275 | hex_digit 276 | = [0-9a-fA-F] 277 | 278 | _ "whitespace" 279 | = whitespace* 280 | 281 | whitespace 282 | = [ \t\n\r] 283 | -------------------------------------------------------------------------------- /lib/cube/event.js: -------------------------------------------------------------------------------- 1 | // TODO include the event._id (and define a JSON encoding for ObjectId?) 2 | // TODO allow the event time to change when updating (fix invalidation) 3 | 4 | var mongodb = require("mongodb"), 5 | parser = require("./event-expression"), 6 | tiers = require("./tiers"), 7 | types = require("./types"), 8 | bisect = require("./bisect"), 9 | ObjectID = mongodb.ObjectID; 10 | 11 | var type_re = /^[a-z][a-zA-Z0-9_]+$/, 12 | invalidate = {$set: {i: true}}, 13 | multi = {multi: true}, 14 | metric_options = {capped: true, size: 1e7, autoIndexId: true}; 15 | 16 | // When streaming events, we should allow a delay for events to arrive, or else 17 | // we risk skipping events that arrive after their event.time. This delay can be 18 | // customized by specifying a `delay` property as part of the request. 19 | var streamDelayDefault = 5000, 20 | streamInterval = 1000; 21 | 22 | // How frequently to invalidate metrics after receiving events. 23 | var invalidateInterval = 5000; 24 | 25 | exports.putter = function(db) { 26 | var collection = types(db), 27 | knownByType = {}, 28 | eventsToSaveByType = {}, 29 | timesToInvalidateByTierByType = {}; 30 | 31 | function putter(request, callback) { 32 | var time = "time" in request ? new Date(request.time) : new Date(), 33 | type = request.type; 34 | 35 | // Validate the date and type. 36 | if (!type_re.test(type)) return callback({error: "invalid type"}), -1; 37 | if (isNaN(time)) return callback({error: "invalid time"}), -1; 38 | 39 | // If an id is specified, promote it to Mongo's primary key. 40 | var event = {t: time, d: request.data}; 41 | if ("id" in request) event._id = request.id; 42 | 43 | // If this is a known event type, save immediately. 44 | if (type in knownByType) return save(type, event); 45 | 46 | // If someone is already creating the event collection for this new type, 47 | // then append this event to the queue for later save. 48 | if (type in eventsToSaveByType) return eventsToSaveByType[type].push(event); 49 | 50 | // Otherwise, it's up to us to see if the collection exists, verify the 51 | // associated indexes, create the corresponding metrics collection, and save 52 | // any events that have queued up in the interim! 53 | 54 | // First add the new event to the queue. 55 | eventsToSaveByType[type] = [event]; 56 | 57 | // If the events collection exists, then we assume the metrics & indexes do 58 | // too. Otherwise, we must create the required collections and indexes. Note 59 | // that if you want to customize the size of the capped metrics collection, 60 | // or add custom indexes, you can still do all that by hand. 61 | db.collectionNames(type + "_events", function(error, names) { 62 | var events = collection(type).events; 63 | if (names.length) return saveEvents(); 64 | 65 | // Events are indexed by time. 66 | events.ensureIndex({"t": 1}, handle); 67 | 68 | // Create a capped collection for metrics. Three indexes are required: one 69 | // for finding metrics, one (_id) for updating, and one for invalidation. 70 | db.createCollection(type + "_metrics", metric_options, function(error, metrics) { 71 | handle(error); 72 | metrics.ensureIndex({"i": 1, "_id.e": 1, "_id.l": 1, "_id.t": 1}, handle); 73 | metrics.ensureIndex({"i": 1, "_id.l": 1, "_id.t": 1}, handle); 74 | saveEvents(); 75 | }); 76 | 77 | // Save any pending events to the new collection. 78 | function saveEvents() { 79 | knownByType[type] = true; 80 | eventsToSaveByType[type].forEach(function(event) { save(type, event); }); 81 | delete eventsToSaveByType[type]; 82 | } 83 | }); 84 | } 85 | 86 | // Save the event of the specified type, and queue invalidation of any cached 87 | // metrics associated with this event type and time. 88 | // 89 | // We don't invalidate the events immediately. This would cause many redundant 90 | // updates when many events are received simultaneously. Also, having a short 91 | // delay between saving the event and invalidating the metrics reduces the 92 | // likelihood of a race condition between when the events are read by the 93 | // evaluator and when the newly-computed metrics are saved. 94 | function save(type, event) { 95 | collection(type).events.save(event, handle); 96 | queueInvalidation(type, event); 97 | } 98 | 99 | // Schedule deferred invalidation of metrics for this type. 100 | // For each type and tier, track the metric times to invalidate. 101 | // The times are kept in sorted order for bisection. 102 | function queueInvalidation(type, event) { 103 | var timesToInvalidateByTier = timesToInvalidateByTierByType[type], 104 | time = event.t; 105 | if (timesToInvalidateByTier) { 106 | for (var tier in tiers) { 107 | var tierTimes = timesToInvalidateByTier[tier], 108 | tierTime = tiers[tier].floor(time), 109 | i = bisect(tierTimes, tierTime); 110 | if (i >= tierTimes.length) tierTimes.push(tierTime); 111 | else if (tierTimes[i] > tierTime) tierTimes.splice(i, 0, tierTime); 112 | } 113 | } else { 114 | timesToInvalidateByTier = timesToInvalidateByTierByType[type] = {}; 115 | for (var tier in tiers) { 116 | timesToInvalidateByTier[tier] = [tiers[tier].floor(time)]; 117 | } 118 | } 119 | } 120 | 121 | // Process any deferred metric invalidations, flushing the queues. Note that 122 | // the queue (timesToInvalidateByTierByType) is copied-on-write, so while the 123 | // previous batch of events are being invalidated, new events can arrive. 124 | setInterval(function() { 125 | for (var type in timesToInvalidateByTierByType) { 126 | var metrics = collection(type).metrics, 127 | timesToInvalidateByTier = timesToInvalidateByTierByType[type]; 128 | for (var tier in tiers) { 129 | metrics.update({ 130 | i: false, 131 | "_id.l": +tier, 132 | "_id.t": {$in: timesToInvalidateByTier[tier]} 133 | }, invalidate, multi, handle); 134 | } 135 | flushed = true; 136 | } 137 | timesToInvalidateByTierByType = {}; // copy-on-write 138 | }, invalidateInterval); 139 | 140 | return putter; 141 | }; 142 | 143 | exports.getter = function(db) { 144 | var collection = types(db), 145 | streamsBySource = {}; 146 | 147 | function getter(request, callback) { 148 | var stream = !("stop" in request), 149 | delay = "delay" in request ? +request.delay : streamDelayDefault, 150 | start = new Date(request.start), 151 | stop = stream ? new Date(Date.now() - delay) : new Date(request.stop); 152 | 153 | // Validate the dates. 154 | if (isNaN(start)) return callback({error: "invalid start"}), -1; 155 | if (isNaN(stop)) return callback({error: "invalid stop"}), -1; 156 | 157 | // Parse the expression. 158 | var expression; 159 | try { 160 | expression = parser.parse(request.expression); 161 | } catch (error) { 162 | return callback({error: "invalid expression"}), -1; 163 | } 164 | 165 | // Set an optional limit on the number of events to return. 166 | var options = {sort: {t: -1}, batchSize: 1000}; 167 | if ("limit" in request) options.limit = +request.limit; 168 | 169 | // Copy any expression filters into the query object. 170 | var filter = {t: {$gte: start, $lt: stop}}; 171 | expression.filter(filter); 172 | 173 | // Request any needed fields. 174 | var fields = {t: 1}; 175 | expression.fields(fields); 176 | 177 | // Query for the desired events. 178 | function query(callback) { 179 | collection(expression.type).events.find(filter, fields, options, function(error, cursor) { 180 | handle(error); 181 | cursor.each(function(error, event) { 182 | 183 | // If the callback is closed (i.e., if the WebSocket connection was 184 | // closed), then abort the query. Note that closing the cursor mid- 185 | // loop causes an error, which we subsequently ignore! 186 | if (callback.closed) return cursor.close(); 187 | 188 | handle(error); 189 | 190 | // A null event indicates that there are no more results. 191 | if (event) callback({id: event._id instanceof ObjectID ? undefined : event._id, time: event.t, data: event.d}); 192 | else callback(null); 193 | }); 194 | }); 195 | } 196 | 197 | // For streaming queries, share streams for efficient polling. 198 | if (stream) { 199 | var streams = streamsBySource[expression.source]; 200 | 201 | // If there is an existing stream to attach to, backfill the initial set 202 | // of results to catch the client up to the stream. Add the new callback 203 | // to a queue, so that when the shared stream finishes its current poll, 204 | // it begins notifying the new client. Note that we don't pass the null 205 | // (end terminator) to the callback, because more results are to come! 206 | if (streams) { 207 | filter.t.$lt = streams.time; 208 | streams.waiting.push(callback); 209 | query(function(event) { if (event) callback(event); }); 210 | } 211 | 212 | // Otherwise, we're creating a new stream, so we're responsible for 213 | // starting the polling loop. This means notifying active callbacks, 214 | // detecting when active callbacks are closed, advancing the time window, 215 | // and moving waiting clients to active clients. 216 | else { 217 | streams = streamsBySource[expression.source] = {time: stop, waiting: [], active: [callback]}; 218 | (function poll() { 219 | query(function(event) { 220 | 221 | // If there's an event, send it to all active, open clients. 222 | if (event) { 223 | streams.active.forEach(function(callback) { 224 | if (!callback.closed) callback(event); 225 | }); 226 | } 227 | 228 | // Otherwise, we've reached the end of a poll, and it's time to 229 | // merge the waiting callbacks into the active callbacks. Advance 230 | // the time range, and set a timeout for the next poll. 231 | else { 232 | streams.active = streams.active.concat(streams.waiting).filter(open); 233 | streams.waiting = []; 234 | 235 | // If no clients remain, then it's safe to delete the shared 236 | // stream, and we'll no longer be responsible for polling. 237 | if (!streams.active.length) { 238 | delete streamsBySource[expression.source]; 239 | return; 240 | } 241 | 242 | filter.t.$gte = streams.time; 243 | filter.t.$lt = streams.time = new Date(Date.now() - delay); 244 | setTimeout(poll, streamInterval); 245 | } 246 | }); 247 | })(); 248 | } 249 | } 250 | 251 | // For non-streaming queries, just send the single batch! 252 | else query(callback); 253 | } 254 | 255 | getter.close = function(callback) { 256 | callback.closed = true; 257 | }; 258 | 259 | return getter; 260 | }; 261 | 262 | function handle(error) { 263 | if (error) throw error; 264 | } 265 | 266 | function open(callback) { 267 | return !callback.closed; 268 | } 269 | -------------------------------------------------------------------------------- /test/metric-expression-test.js: -------------------------------------------------------------------------------- 1 | var vows = require("vows"), 2 | assert = require("assert"), 3 | parser = require("../lib/cube/metric-expression"); 4 | 5 | var suite = vows.describe("metric-expression"); 6 | 7 | suite.addBatch({ 8 | 9 | "a simple unary expression, sum(test)": { 10 | topic: parser.parse("sum(test)"), 11 | "is unary (has no associated binary operator)": function(e) { 12 | assert.isUndefined(e.op); 13 | }, 14 | "has the expected event type, test": function(e) { 15 | assert.equal(e.type, "test"); 16 | }, 17 | "does not load any event data fields": function(e) { 18 | var fields = {}; 19 | e.fields(fields); 20 | assert.deepEqual(fields, {}); 21 | }, 22 | "does not apply any event data filters": function(e) { 23 | var filter = {}; 24 | e.filter(filter); 25 | assert.deepEqual(filter, {}); 26 | }, 27 | "has the expected reduce": function(e) { 28 | assert.equal(e.reduce, "sum"); 29 | }, 30 | "has the expected source": function(e) { 31 | assert.equal(e.source, "sum(test)"); 32 | }, 33 | "returns the value one for any event": function(e) { 34 | assert.equal(e.value(), 1); 35 | } 36 | }, 37 | 38 | "an expression with a simple data accessor": { 39 | topic: parser.parse("sum(test(i))"), 40 | "loads the specified event data field": function(e) { 41 | var fields = {}; 42 | e.fields(fields); 43 | assert.deepEqual(fields, {"d.i": 1}); 44 | }, 45 | "ignores events that do not have the specified field": function(e) { 46 | var filter = {}; 47 | e.filter(filter); 48 | assert.deepEqual(filter, {"d.i": {$exists: true}}); 49 | }, 50 | "returns the specified field value": function(e) { 51 | assert.equal(e.value({d: {i: 42}}), 42); 52 | assert.equal(e.value({d: {i: -1}}), -1); 53 | }, 54 | "has the expected source": function(e) { 55 | assert.equal(e.source, "sum(test(i))"); 56 | } 57 | }, 58 | 59 | "an expression with a compound data accessor": { 60 | topic: parser.parse("sum(test(i + i * i - 2))"), 61 | "loads the specified event data field": function(e) { 62 | var fields = {}; 63 | e.fields(fields); 64 | assert.deepEqual(fields, {"d.i": 1}); 65 | }, 66 | "ignores events that do not have the specified field": function(e) { 67 | var filter = {}; 68 | e.filter(filter); 69 | assert.deepEqual(filter, {"d.i": {$exists: true}}); 70 | }, 71 | "computes the specified value expression": function(e) { 72 | assert.equal(e.value({d: {i: 42}}), 1804); 73 | assert.equal(e.value({d: {i: -1}}), -2); 74 | }, 75 | "has the expected source": function(e) { 76 | assert.equal(e.source, "sum(test(i + i * i - 2))"); 77 | } 78 | }, 79 | 80 | "an expression with a data accessor and a filter on the same field": { 81 | topic: parser.parse("sum(test(i).gt(i, 42))"), 82 | "loads the specified event data field": function(e) { 83 | var fields = {}; 84 | e.fields(fields); 85 | assert.deepEqual(fields, {"d.i": 1}); 86 | }, 87 | "only filters using the explicit filter; existence is implied": function(e) { 88 | var filter = {}; 89 | e.filter(filter); 90 | assert.deepEqual(filter, {"d.i": {$gt: 42}}); 91 | }, 92 | "has the expected source": function(e) { 93 | assert.equal(e.source, "sum(test(i).gt(i, 42))"); 94 | } 95 | }, 96 | 97 | "an expression with filters on different fields": { 98 | topic: parser.parse("sum(test.gt(i, 42).eq(j, \"foo\"))"), 99 | "does not load fields that are only filtered": function(e) { 100 | var fields = {}; 101 | e.fields(fields); 102 | assert.deepEqual(fields, {}); 103 | }, 104 | "has the expected filters on each specified field": function(e) { 105 | var filter = {}; 106 | e.filter(filter); 107 | assert.deepEqual(filter, {"d.i": {$gt: 42}, "d.j": "foo"}); 108 | }, 109 | "has the expected source": function(e) { 110 | assert.equal(e.source, "sum(test.gt(i, 42).eq(j, \"foo\"))"); 111 | } 112 | }, 113 | 114 | "an expression with an object data accessor": { 115 | topic: parser.parse("sum(test(i.j))"), 116 | "loads the specified event data field": function(e) { 117 | var fields = {}; 118 | e.fields(fields); 119 | assert.deepEqual(fields, {"d.i.j": 1}); 120 | }, 121 | "ignores events that do not have the specified field": function(e) { 122 | var filter = {}; 123 | e.filter(filter); 124 | assert.deepEqual(filter, {"d.i.j": {$exists: true}}); 125 | }, 126 | "computes the specified value expression": function(e) { 127 | assert.equal(e.value({d: {i: {j: 42}}}), 42); 128 | assert.equal(e.value({d: {i: {j: -1}}}), -1); 129 | }, 130 | "has the expected source": function(e) { 131 | assert.equal(e.source, "sum(test(i.j))"); 132 | } 133 | }, 134 | 135 | "an expression with an array data accessor": { 136 | topic: parser.parse("sum(test(i[0]))"), 137 | "loads the specified event data field": function(e) { 138 | var fields = {}; 139 | e.fields(fields); 140 | assert.deepEqual(fields, {"d.i": 1}); 141 | }, 142 | "ignores events that do not have the specified field": function(e) { 143 | var filter = {}; 144 | e.filter(filter); 145 | assert.deepEqual(filter, {"d.i": {$exists: true}}); 146 | }, 147 | "computes the specified value expression": function(e) { 148 | assert.equal(e.value({d: {i: [42]}}), 42); 149 | assert.equal(e.value({d: {i: [-1]}}), -1); 150 | }, 151 | "has the expected source": function(e) { 152 | assert.equal(e.source, "sum(test(i[0]))"); 153 | } 154 | }, 155 | 156 | "an expression with an elaborate data accessor": { 157 | topic: parser.parse("sum(test(i.j[0].k))"), 158 | "loads the specified event data field": function(e) { 159 | var fields = {}; 160 | e.fields(fields); 161 | assert.deepEqual(fields, {"d.i.j.k": 1}); 162 | }, 163 | "ignores events that do not have the specified field": function(e) { 164 | var filter = {}; 165 | e.filter(filter); 166 | assert.deepEqual(filter, {"d.i.j.k": {$exists: true}}); 167 | }, 168 | "computes the specified value expression": function(e) { 169 | assert.equal(e.value({d: {i: {j: [{k: 42}]}}}), 42); 170 | assert.equal(e.value({d: {i: {j: [{k: -1}]}}}), -1); 171 | }, 172 | "has the expected source": function(e) { 173 | assert.equal(e.source, "sum(test(i.j[0].k))"); 174 | } 175 | }, 176 | 177 | "a compound expression of two unary expressions": { 178 | topic: parser.parse("sum(foo(2)) + sum(bar(3))"), 179 | "is compound (has an associated binary operator)": function(e) { 180 | assert.equal(e.op.name, "add"); 181 | }, 182 | "has the expected left expression": function(e) { 183 | var filter = {}, fields = {}; 184 | e.left.filter(filter); 185 | e.left.fields(fields); 186 | assert.deepEqual(filter, {}); 187 | assert.deepEqual(fields, {}); 188 | assert.equal(e.left.source, "sum(foo(2))"); 189 | assert.equal(e.left.type, "foo"); 190 | assert.equal(e.left.reduce, "sum"); 191 | assert.equal(e.left.value(), 2); 192 | }, 193 | "has the expected right expression": function(e) { 194 | var filter = {}, fields = {}; 195 | e.right.filter(filter); 196 | e.right.fields(fields); 197 | assert.deepEqual(filter, {}); 198 | assert.deepEqual(fields, {}); 199 | assert.equal(e.right.source, "sum(bar(3))"); 200 | assert.equal(e.right.type, "bar"); 201 | assert.equal(e.right.reduce, "sum"); 202 | assert.equal(e.right.value(), 3); 203 | }, 204 | "does not have a source": function(e) { 205 | assert.isUndefined(e.source); 206 | }, 207 | "computes the specified value expression": function(e) { 208 | assert.equal(e.op(2, 3), 5) 209 | } 210 | }, 211 | 212 | "a compound expression of three unary expressions": { 213 | topic: parser.parse("sum(foo(2)) + median(bar(3)) + max(baz(qux))"), 214 | "has the expected subexpression sources": function(e) { 215 | assert.isUndefined(e.source); 216 | assert.equal(e.left.source, "sum(foo(2))"); 217 | assert.isUndefined(e.right.source); 218 | assert.equal(e.right.left.source, "median(bar(3))"); 219 | assert.equal(e.right.right.source, "max(baz(qux))"); 220 | } 221 | }, 222 | 223 | "a negated unary expression": { 224 | topic: parser.parse("-sum(foo)"), 225 | "negates the specified value expression": function(e) { 226 | assert.equal(e.value(), -1) 227 | }, 228 | "has the expected source": function(e) { 229 | assert.equal(e.source, "-sum(foo)"); 230 | } 231 | }, 232 | 233 | "constant expressions": { 234 | topic: parser.parse("-4"), 235 | "has a constant value": function(e) { 236 | assert.equal(e.value(), -4) 237 | }, 238 | "does not have a source": function(e) { 239 | assert.isUndefined(e.source); 240 | } 241 | }, 242 | 243 | "filters": { 244 | "multiple filters on the same field are combined": function() { 245 | var filter = {}; 246 | parser.parse("sum(test.gt(i, 42).le(i, 52))").filter(filter); 247 | assert.deepEqual(filter, {"d.i": {$gt: 42, $lte: 52}}); 248 | }, 249 | "given range and exact filters, range filters are ignored": function() { 250 | var filter = {}; 251 | parser.parse("sum(test.gt(i, 42).eq(i, 52))").filter(filter); 252 | assert.deepEqual(filter, {"d.i": 52}); 253 | }, 254 | "given exact and range filters, range filters are ignored": function() { 255 | var filter = {}; 256 | parser.parse("sum(test.eq(i, 52).gt(i, 42))").filter(filter); 257 | assert.deepEqual(filter, {"d.i": 52}); 258 | }, 259 | "the eq filter results in a simple query filter": function() { 260 | var filter = {}; 261 | parser.parse("sum(test.eq(i, 42))").filter(filter); 262 | assert.deepEqual(filter, {"d.i": 42}); 263 | }, 264 | "the gt filter results in a $gt query filter": function(e) { 265 | var filter = {}; 266 | parser.parse("sum(test.gt(i, 42))").filter(filter); 267 | assert.deepEqual(filter, {"d.i": {$gt: 42}}); 268 | }, 269 | "the ge filter results in a $gte query filter": function(e) { 270 | var filter = {}; 271 | parser.parse("sum(test.ge(i, 42))").filter(filter); 272 | assert.deepEqual(filter, {"d.i": {$gte: 42}}); 273 | }, 274 | "the lt filter results in an $lt query filter": function(e) { 275 | var filter = {}; 276 | parser.parse("sum(test.lt(i, 42))").filter(filter); 277 | assert.deepEqual(filter, {"d.i": {$lt: 42}}); 278 | }, 279 | "the le filter results in an $lte query filter": function(e) { 280 | var filter = {}; 281 | parser.parse("sum(test.le(i, 42))").filter(filter); 282 | assert.deepEqual(filter, {"d.i": {$lte: 42}}); 283 | }, 284 | "the ne filter results in an $ne query filter": function(e) { 285 | var filter = {}; 286 | parser.parse("sum(test.ne(i, 42))").filter(filter); 287 | assert.deepEqual(filter, {"d.i": {$ne: 42}}); 288 | }, 289 | "the re filter results in a $regex query filter": function(e) { 290 | var filter = {}; 291 | parser.parse("sum(test.re(i, \"foo\"))").filter(filter); 292 | assert.deepEqual(filter, {"d.i": {$regex: "foo"}}); 293 | }, 294 | "the in filter results in a $in query filter": function(e) { 295 | var filter = {}; 296 | parser.parse("sum(test.in(i, [\"foo\", 42]))").filter(filter); 297 | assert.deepEqual(filter, {"d.i": {$in: ["foo", 42]}}); 298 | } 299 | } 300 | 301 | }); 302 | 303 | suite.export(module); 304 | -------------------------------------------------------------------------------- /static/cubism.v1.min.js: -------------------------------------------------------------------------------- 1 | (function(a){function d(a){return a}function e(){}function h(a){return Math.floor(a/1e3)}function i(a){var b=a.indexOf("|"),c=a.substring(0,b),d=c.lastIndexOf(","),e=c.lastIndexOf(",",d-1),f=c.lastIndexOf(",",e-1),g=c.substring(f+1,e)*1e3,h=c.substring(d+1)*1e3;return a.substring(b+1).split(",").slice(1).map(function(a){return+a})}function j(a){if(!(a instanceof e))throw new Error("invalid context");this.context=a}function m(a,b){return function(c,d,e,f){a(new Date(+c+b),new Date(+d+b),e,f)}}function n(a,b){j.call(this,a),b=+b;var c=b+"";this.valueOf=function(){return b},this.toString=function(){return c}}function p(a,b){function c(b,c){if(c instanceof j){if(b.context!==c.context)throw new Error("mismatch context")}else c=new n(b.context,c);j.call(this,b.context),this.left=b,this.right=c,this.toString=function(){return b+" "+a+" "+c}}var d=c.prototype=Object.create(j.prototype);return d.valueAt=function(a){return b(this.left.valueAt(a),this.right.valueAt(a))},d.shift=function(a){return new c(this.left.shift(a),this.right.shift(a))},d.on=function(a,b){return arguments.length<2?this.left.on(a):(this.left.on(a,b),this.right.on(a,b),this)},function(a){return new c(this,a)}}function s(a){return a&16777214}function t(a){return(a+1&16777214)-1}function x(a){a.style("position","absolute").style("top",0).style("bottom",0).style("width","1px").style("pointer-events","none")}function y(a){return a+"px"}var b=a.cubism={version:"1.3.0"},c=0;b.option=function(a,c){var d=b.options(a);return d.length?d[0]:c},b.options=function(a,b){var c=location.search.substring(1).split("&"),d=[],e=-1,f=c.length,g;while(++e0&&a.focus(--o);break;case 39:o==null&&(o=d-2),oe&&(e=c);return[d,e]},k.on=function(a,b){return arguments.length<2?null:this},k.shift=function(){return this},k.on=function(){return arguments.length<2?null:this},f.metric=function(a,b){function r(b,c){var d=Math.min(k,Math.round((b-g)/i));if(!d||q)return;q=!0,d=Math.min(k,d+l);var f=new Date(c-d*i);a(f,c,i,function(a,b){q=!1;if(a)return console.warn(a);var d=isFinite(g)?Math.round((f-g)/i):0;for(var h=0,j=b.length;h1&&(e.toString=function(){return b}),e};var l=6,o=n.prototype=Object.create(j.prototype);o.valueAt=function(){return+this},o.extent=function(){return[+this,+this]},k.add=p("+",function(a,b){return a+b}),k.subtract=p("-",function(a,b){return a-b}),k.multiply=p("*",function(a,b){return a*b}),k.divide=p("/",function(a,b){return a/b}),f.horizon=function(){function o(o){o.on("mousemove.horizon",function(){a.focus(Math.round(d3.mouse(this)[0]))}).on("mouseout.horizon",function(){a.focus(null)}),o.append("canvas").attr("width",f).attr("height",g),o.append("span").attr("class","title").text(k),o.append("span").attr("class","value"),o.each(function(k,o){function B(c,d){w.save();var i=r.extent();A=i.every(isFinite),t!=null&&(i=t);var j=0,k=Math.max(-i[0],i[1]);if(this===a){if(k==y){j=f-l;var m=(c-u)/v;if(m=0)continue;w.fillRect(x,h(-C),1,q-h(-C))}}}w.restore()}function C(a){a==null&&(a=f-1);var b=r.valueAt(a);x.datum(b).text(isNaN(b)?null:m)}var p=this,q=++c,r=typeof i=="function"?i.call(p,k,o):i,s=typeof n=="function"?n.call(p,k,o):n,t=typeof j=="function"?j.call(p,k,o):j,u=-Infinity,v=a.step(),w=d3.select(p).select("canvas"),x=d3.select(p).select(".value"),y,z=s.length>>1,A;w.datum({id:q,metric:r}),w=w.node().getContext("2d"),a.on("change.horizon-"+q,B),a.on("focus.horizon-"+q,C),r.on("change.horizon-"+q,function(a,b){B(a,b),C(),A&&r.on("change.horizon-"+q,d)})})}var a=this,b="offset",e=document.createElement("canvas"),f=e.width=a.size(),g=e.height=30,h=d3.scale.linear().interpolate(d3.interpolateRound),i=d,j=null,k=d,m=d3.format(".2s"),n=["#08519c","#3182bd","#6baed6","#bdd7e7","#bae4b3","#74c476","#31a354","#006d2c"];return o.remove=function(b){function c(b){b.metric.on("change.horizon-"+b.id,null),a.on("change.horizon-"+b.id,null),a.on("focus.horizon-"+b.id,null)}b.on("mousemove.horizon",null).on("mouseout.horizon",null),b.selectAll("canvas").each(c).remove(),b.selectAll(".title,.value").remove()},o.mode=function(a){return arguments.length?(b=a+"",o):b},o.height=function(a){return arguments.length?(e.height=g=+a,o):g},o.metric=function(a){return arguments.length?(i=a,o):i},o.scale=function(a){return arguments.length?(h=a,o):h},o.extent=function(a){return arguments.length?(j=a,o):j},o.title=function(a){return arguments.length?(k=a,o):k},o.format=function(a){return arguments.length?(m=a,o):m},o.colors=function(a){return arguments.length?(n=a,o):n},o},f.comparison=function(){function o(o){o.on("mousemove.comparison",function(){a.focus(Math.round(d3.mouse(this)[0]))}).on("mouseout.comparison",function(){a.focus(null)}),o.append("canvas").attr("width",b).attr("height",e),o.append("span").attr("class","title").text(j),o.append("span").attr("class","value primary"),o.append("span").attr("class","value change"),o.each(function(j,o){function B(c,d){x.save(),x.clearRect(0,0,b,e);var g=r.extent(),h=u.extent(),i=v==null?g:v;f.domain(i).range([e,0]),A=g.concat(h).every(isFinite);var j=c/a.step()&1?t:s;x.fillStyle=m[2];for(var k=0,l=b;kp&&x.fillRect(j(k),p,1,o-p)}x.fillStyle=m[3];for(k=0;kp&&x.fillRect(j(k),o-n,1,n)}x.restore()}function C(a){a==null&&(a=b-1);var c=r.valueAt(a),d=u.valueAt(a),e=(c-d)/d;y.datum(c).text(isNaN(c)?null:k),z.datum(e).text(isNaN(e)?null:l).attr("class","value change "+(e>0?"positive":e<0?"negative":""))}function D(a,b){B(a,b),C(),A&&(r.on("change.comparison-"+q,d),u.on("change.comparison-"+q,d))}var p=this,q=++c,r=typeof g=="function"?g.call(p,j,o):g,u=typeof h=="function"?h.call(p,j,o):h,v=typeof i=="function"?i.call(p,j,o):i,w=d3.select(p),x=w.select("canvas"),y=w.select(".value.primary"),z=w.select(".value.change"),A;x.datum({id:q,primary:r,secondary:u}),x=x.node().getContext("2d"),r.on("change.comparison-"+q,D),u.on("change.comparison-"+q,D),a.on("change.comparison-"+q,B),a.on("focus.comparison-"+q,C)})}var a=this,b=a.size(),e=120,f=d3.scale.linear().interpolate(d3.interpolateRound),g=function(a){return a[0]},h=function(a){return a[1]},i=null,j=d,k=q,l=r,m=["#9ecae1","#225b84","#a1d99b","#22723a"],n=1.5;return o.remove=function(b){function c(b){b.primary.on("change.comparison-"+b.id,null),b.secondary.on("change.comparison-"+b.id,null),a.on("change.comparison-"+b.id,null),a.on("focus.comparison-"+b.id,null)}b.on("mousemove.comparison",null).on("mouseout.comparison",null),b.selectAll("canvas").each(c).remove(),b.selectAll(".title,.value").remove()},o.height=function(a){return arguments.length?(e=+a,o):e},o.primary=function(a){return arguments.length?(g=a,o):g},o.secondary=function(a){return arguments.length?(h=a,o):h},o.scale=function(a){return arguments.length?(f=a,o):f},o.extent=function(a){return arguments.length?(i=a,o):i},o.title=function(a){return arguments.length?(j=a,o):j},o.formatPrimary=function(a){return arguments.length?(k=a,o):k},o.formatChange=function(a){return arguments.length?(l=a,o):l},o.colors=function(a){return arguments.length?(m=a,o):m},o.strokeWidth=function(a){return arguments.length?(n=a,o):n},o};var q=d3.format(".2s"),r=d3.format("+.0%");f.axis=function(){function f(g){var h=++c,i,j=g.append("svg").datum({id:h}).attr("width",a.size()).attr("height",Math.max(28,-f.tickSize())).append("g").attr("transform","translate(0,"+(d.orient()==="top"?27:4)+")").call(d);a.on("change.axis-"+h,function(){j.call(d),i||(i=d3.select(j.node().appendChild(j.selectAll("text").node().cloneNode(!0))).style("display","none").text(null))}),a.on("focus.axis-"+h,function(a){if(i)if(a==null)i.style("display","none"),j.selectAll("text").style("fill-opacity",null);else{i.style("display",null).attr("x",a).text(e(b.invert(a)));var c=i.node().getComputedTextLength()+6;j.selectAll("text").style("fill-opacity",function(d){return Math.abs(b(d)-a) 0) context.focus(--focus); 141 | break; 142 | case 39: // right 143 | if (focus == null) focus = size - 2; 144 | if (focus < size - 1) context.focus(++focus); 145 | break; 146 | default: return; 147 | } 148 | d3.event.preventDefault(); 149 | }); 150 | 151 | return update(); 152 | }; 153 | 154 | function cubism_context() {} 155 | 156 | var cubism_contextPrototype = cubism.context.prototype = cubism_context.prototype; 157 | 158 | cubism_contextPrototype.constant = function(value) { 159 | return new cubism_metricConstant(this, +value); 160 | }; 161 | cubism_contextPrototype.cube = function(host) { 162 | if (!arguments.length) host = ""; 163 | var source = {}, 164 | context = this; 165 | 166 | source.metric = function(expression) { 167 | return context.metric(function(start, stop, step, callback) { 168 | d3.json(host + "/1.0/metric" 169 | + "?expression=" + encodeURIComponent(expression) 170 | + "&start=" + cubism_cubeFormatDate(start) 171 | + "&stop=" + cubism_cubeFormatDate(stop) 172 | + "&step=" + step, function(data) { 173 | if (!data) return callback(new Error("unable to load data")); 174 | callback(null, data.map(function(d) { return d.value; })); 175 | }); 176 | }, expression += ""); 177 | }; 178 | 179 | // Returns the Cube host. 180 | source.toString = function() { 181 | return host; 182 | }; 183 | 184 | return source; 185 | }; 186 | 187 | var cubism_cubeFormatDate = d3.time.format.iso; 188 | cubism_contextPrototype.graphite = function(host) { 189 | if (!arguments.length) host = ""; 190 | var source = {}, 191 | context = this; 192 | 193 | source.metric = function(expression) { 194 | var sum = "sum"; 195 | 196 | var metric = context.metric(function(start, stop, step, callback) { 197 | var target = expression; 198 | 199 | // Apply the summarize, if necessary. 200 | if (step !== 1e4) target = "summarize(" + target + ",'" 201 | + (!(step % 36e5) ? step / 36e5 + "hour" : !(step % 6e4) ? step / 6e4 + "min" : step / 1e3 + "sec") 202 | + "','" + sum + "')"; 203 | 204 | d3.text(host + "/render?format=raw" 205 | + "&target=" + encodeURIComponent("alias(" + target + ",'')") 206 | + "&from=" + cubism_graphiteFormatDate(start - 2 * step) // off-by-two? 207 | + "&until=" + cubism_graphiteFormatDate(stop - 1000), function(text) { 208 | if (!text) return callback(new Error("unable to load data")); 209 | callback(null, cubism_graphiteParse(text)); 210 | }); 211 | }, expression += ""); 212 | 213 | metric.summarize = function(_) { 214 | sum = _; 215 | return metric; 216 | }; 217 | 218 | return metric; 219 | }; 220 | 221 | source.find = function(pattern, callback) { 222 | d3.json(host + "/metrics/find?format=completer" 223 | + "&query=" + encodeURIComponent(pattern), function(result) { 224 | if (!result) return callback(new Error("unable to find metrics")); 225 | callback(null, result.metrics.map(function(d) { return d.path; })); 226 | }); 227 | }; 228 | 229 | // Returns the graphite host. 230 | source.toString = function() { 231 | return host; 232 | }; 233 | 234 | return source; 235 | }; 236 | 237 | // Graphite understands seconds since UNIX epoch. 238 | function cubism_graphiteFormatDate(time) { 239 | return Math.floor(time / 1000); 240 | } 241 | 242 | // Helper method for parsing graphite's raw format. 243 | function cubism_graphiteParse(text) { 244 | var i = text.indexOf("|"), 245 | meta = text.substring(0, i), 246 | c = meta.lastIndexOf(","), 247 | b = meta.lastIndexOf(",", c - 1), 248 | a = meta.lastIndexOf(",", b - 1), 249 | start = meta.substring(a + 1, b) * 1000, 250 | step = meta.substring(c + 1) * 1000; 251 | return text 252 | .substring(i + 1) 253 | .split(",") 254 | .slice(1) // the first value is always None? 255 | .map(function(d) { return +d; }); 256 | } 257 | cubism_contextPrototype.gangliaWeb = function(config) { 258 | var host = '', 259 | uriPathPrefix = '/ganglia2/'; 260 | 261 | if (arguments.length) { 262 | if (config.host) { 263 | host = config.host; 264 | } 265 | 266 | if (config.uriPathPrefix) { 267 | uriPathPrefix = config.uriPathPrefix; 268 | 269 | /* Add leading and trailing slashes, as appropriate. */ 270 | if( uriPathPrefix[0] != '/' ) { 271 | uriPathPrefix = '/' + uriPathPrefix; 272 | } 273 | 274 | if( uriPathPrefix[uriPathPrefix.length - 1] != '/' ) { 275 | uriPathPrefix += '/'; 276 | } 277 | } 278 | } 279 | 280 | var source = {}, 281 | context = this; 282 | 283 | source.metric = function(metricInfo) { 284 | 285 | /* Store the members from metricInfo into local variables. */ 286 | var clusterName = metricInfo.clusterName, 287 | metricName = metricInfo.metricName, 288 | hostName = metricInfo.hostName, 289 | isReport = metricInfo.isReport || false, 290 | titleGenerator = metricInfo.titleGenerator || 291 | /* Reasonable (not necessarily pretty) default for titleGenerator. */ 292 | function(unusedMetricInfo) { 293 | /* unusedMetricInfo is, well, unused in this default case. */ 294 | return ('clusterName:' + clusterName + 295 | ' metricName:' + metricName + 296 | (hostName ? ' hostName:' + hostName : '')); 297 | }, 298 | onChangeCallback = metricInfo.onChangeCallback; 299 | 300 | /* Default to plain, simple metrics. */ 301 | var metricKeyName = isReport ? 'g' : 'm'; 302 | 303 | var gangliaWebMetric = context.metric(function(start, stop, step, callback) { 304 | 305 | function constructGangliaWebRequestQueryParams() { 306 | return ('c=' + clusterName + 307 | '&' + metricKeyName + '=' + metricName + 308 | (hostName ? '&h=' + hostName : '') + 309 | '&cs=' + start/1000 + '&ce=' + stop/1000 + '&step=' + step/1000 + '&graphlot=1'); 310 | } 311 | 312 | d3.json(host + uriPathPrefix + 'graph.php?' + constructGangliaWebRequestQueryParams(), 313 | function(result) { 314 | if( !result ) { 315 | return callback(new Error("Unable to fetch GangliaWeb data")); 316 | } 317 | 318 | callback(null, result[0].data); 319 | }); 320 | 321 | }, titleGenerator(metricInfo)); 322 | 323 | gangliaWebMetric.toString = function() { 324 | return titleGenerator(metricInfo); 325 | }; 326 | 327 | /* Allow users to run their custom code each time a gangliaWebMetric changes. 328 | * 329 | * TODO Consider abstracting away the naked Cubism call, and instead exposing 330 | * a callback that takes in the values array (maybe alongwith the original 331 | * start and stop 'naked' parameters), since it's handy to have the entire 332 | * dataset at your disposal (and users will likely implement onChangeCallback 333 | * primarily to get at this dataset). 334 | */ 335 | if (onChangeCallback) { 336 | gangliaWebMetric.on('change', onChangeCallback); 337 | } 338 | 339 | return gangliaWebMetric; 340 | }; 341 | 342 | // Returns the gangliaWeb host + uriPathPrefix. 343 | source.toString = function() { 344 | return host + uriPathPrefix; 345 | }; 346 | 347 | return source; 348 | }; 349 | 350 | function cubism_metric(context) { 351 | if (!(context instanceof cubism_context)) throw new Error("invalid context"); 352 | this.context = context; 353 | } 354 | 355 | var cubism_metricPrototype = cubism_metric.prototype; 356 | 357 | cubism.metric = cubism_metric; 358 | 359 | cubism_metricPrototype.valueAt = function() { 360 | return NaN; 361 | }; 362 | 363 | cubism_metricPrototype.alias = function(name) { 364 | this.toString = function() { return name; }; 365 | return this; 366 | }; 367 | 368 | cubism_metricPrototype.extent = function() { 369 | var i = 0, 370 | n = this.context.size(), 371 | value, 372 | min = Infinity, 373 | max = -Infinity; 374 | while (++i < n) { 375 | value = this.valueAt(i); 376 | if (value < min) min = value; 377 | if (value > max) max = value; 378 | } 379 | return [min, max]; 380 | }; 381 | 382 | cubism_metricPrototype.on = function(type, listener) { 383 | return arguments.length < 2 ? null : this; 384 | }; 385 | 386 | cubism_metricPrototype.shift = function() { 387 | return this; 388 | }; 389 | 390 | cubism_metricPrototype.on = function() { 391 | return arguments.length < 2 ? null : this; 392 | }; 393 | 394 | cubism_contextPrototype.metric = function(request, name) { 395 | var context = this, 396 | metric = new cubism_metric(context), 397 | id = ".metric-" + ++cubism_id, 398 | start = -Infinity, 399 | stop, 400 | step = context.step(), 401 | size = context.size(), 402 | values = [], 403 | event = d3.dispatch("change"), 404 | listening = 0, 405 | fetching; 406 | 407 | // Prefetch new data into a temporary array. 408 | function prepare(start1, stop) { 409 | var steps = Math.min(size, Math.round((start1 - start) / step)); 410 | if (!steps || fetching) return; // already fetched, or fetching! 411 | fetching = true; 412 | steps = Math.min(size, steps + cubism_metricOverlap); 413 | var start0 = new Date(stop - steps * step); 414 | request(start0, stop, step, function(error, data) { 415 | fetching = false; 416 | if (error) return console.warn(error); 417 | var i = isFinite(start) ? Math.round((start0 - start) / step) : 0; 418 | for (var j = 0, m = data.length; j < m; ++j) values[j + i] = data[j]; 419 | event.change.call(metric, start, stop); 420 | }); 421 | } 422 | 423 | // When the context changes, switch to the new data, ready-or-not! 424 | function beforechange(start1, stop1) { 425 | if (!isFinite(start)) start = start1; 426 | values.splice(0, Math.max(0, Math.min(size, Math.round((start1 - start) / step)))); 427 | start = start1; 428 | stop = stop1; 429 | } 430 | 431 | // 432 | metric.valueAt = function(i) { 433 | return values[i]; 434 | }; 435 | 436 | // 437 | metric.shift = function(offset) { 438 | return context.metric(cubism_metricShift(request, +offset)); 439 | }; 440 | 441 | // 442 | metric.on = function(type, listener) { 443 | if (!arguments.length) return event.on(type); 444 | 445 | // If there are no listeners, then stop listening to the context, 446 | // and avoid unnecessary fetches. 447 | if (listener == null) { 448 | if (event.on(type) != null && --listening == 0) { 449 | context.on("prepare" + id, null).on("beforechange" + id, null); 450 | } 451 | } else { 452 | if (event.on(type) == null && ++listening == 1) { 453 | context.on("prepare" + id, prepare).on("beforechange" + id, beforechange); 454 | } 455 | } 456 | 457 | event.on(type, listener); 458 | 459 | // Notify the listener of the current start and stop time, as appropriate. 460 | // This way, charts can display synchronous metrics immediately. 461 | if (listener != null) { 462 | if (/^change(\.|$)/.test(type)) listener.call(context, start, stop); 463 | } 464 | 465 | return metric; 466 | }; 467 | 468 | // 469 | if (arguments.length > 1) metric.toString = function() { 470 | return name; 471 | }; 472 | 473 | return metric; 474 | }; 475 | 476 | // Number of metric to refetch each period, in case of lag. 477 | var cubism_metricOverlap = 6; 478 | 479 | // Wraps the specified request implementation, and shifts time by the given offset. 480 | function cubism_metricShift(request, offset) { 481 | return function(start, stop, step, callback) { 482 | request(new Date(+start + offset), new Date(+stop + offset), step, callback); 483 | }; 484 | } 485 | function cubism_metricConstant(context, value) { 486 | cubism_metric.call(this, context); 487 | value = +value; 488 | var name = value + ""; 489 | this.valueOf = function() { return value; }; 490 | this.toString = function() { return name; }; 491 | } 492 | 493 | var cubism_metricConstantPrototype = cubism_metricConstant.prototype = Object.create(cubism_metric.prototype); 494 | 495 | cubism_metricConstantPrototype.valueAt = function() { 496 | return +this; 497 | }; 498 | 499 | cubism_metricConstantPrototype.extent = function() { 500 | return [+this, +this]; 501 | }; 502 | function cubism_metricOperator(name, operate) { 503 | 504 | function cubism_metricOperator(left, right) { 505 | if (!(right instanceof cubism_metric)) right = new cubism_metricConstant(left.context, right); 506 | else if (left.context !== right.context) throw new Error("mismatch context"); 507 | cubism_metric.call(this, left.context); 508 | this.left = left; 509 | this.right = right; 510 | this.toString = function() { return left + " " + name + " " + right; }; 511 | } 512 | 513 | var cubism_metricOperatorPrototype = cubism_metricOperator.prototype = Object.create(cubism_metric.prototype); 514 | 515 | cubism_metricOperatorPrototype.valueAt = function(i) { 516 | return operate(this.left.valueAt(i), this.right.valueAt(i)); 517 | }; 518 | 519 | cubism_metricOperatorPrototype.shift = function(offset) { 520 | return new cubism_metricOperator(this.left.shift(offset), this.right.shift(offset)); 521 | }; 522 | 523 | cubism_metricOperatorPrototype.on = function(type, listener) { 524 | if (arguments.length < 2) return this.left.on(type); 525 | this.left.on(type, listener); 526 | this.right.on(type, listener); 527 | return this; 528 | }; 529 | 530 | return function(right) { 531 | return new cubism_metricOperator(this, right); 532 | }; 533 | } 534 | 535 | cubism_metricPrototype.add = cubism_metricOperator("+", function(left, right) { 536 | return left + right; 537 | }); 538 | 539 | cubism_metricPrototype.subtract = cubism_metricOperator("-", function(left, right) { 540 | return left - right; 541 | }); 542 | 543 | cubism_metricPrototype.multiply = cubism_metricOperator("*", function(left, right) { 544 | return left * right; 545 | }); 546 | 547 | cubism_metricPrototype.divide = cubism_metricOperator("/", function(left, right) { 548 | return left / right; 549 | }); 550 | cubism_contextPrototype.horizon = function() { 551 | var context = this, 552 | mode = "offset", 553 | buffer = document.createElement("canvas"), 554 | width = buffer.width = context.size(), 555 | height = buffer.height = 30, 556 | scale = d3.scale.linear().interpolate(d3.interpolateRound), 557 | metric = cubism_identity, 558 | extent = null, 559 | title = cubism_identity, 560 | format = d3.format(".2s"), 561 | colors = ["#08519c","#3182bd","#6baed6","#bdd7e7","#bae4b3","#74c476","#31a354","#006d2c"]; 562 | 563 | function horizon(selection) { 564 | 565 | selection 566 | .on("mousemove.horizon", function() { context.focus(Math.round(d3.mouse(this)[0])); }) 567 | .on("mouseout.horizon", function() { context.focus(null); }); 568 | 569 | selection.append("canvas") 570 | .attr("width", width) 571 | .attr("height", height); 572 | 573 | selection.append("span") 574 | .attr("class", "title") 575 | .text(title); 576 | 577 | selection.append("span") 578 | .attr("class", "value"); 579 | 580 | selection.each(function(d, i) { 581 | var that = this, 582 | id = ++cubism_id, 583 | metric_ = typeof metric === "function" ? metric.call(that, d, i) : metric, 584 | colors_ = typeof colors === "function" ? colors.call(that, d, i) : colors, 585 | extent_ = typeof extent === "function" ? extent.call(that, d, i) : extent, 586 | start = -Infinity, 587 | step = context.step(), 588 | canvas = d3.select(that).select("canvas"), 589 | span = d3.select(that).select(".value"), 590 | max_, 591 | m = colors_.length >> 1, 592 | ready; 593 | 594 | canvas.datum({id: id, metric: metric_}); 595 | canvas = canvas.node().getContext("2d"); 596 | 597 | function change(start1, stop) { 598 | canvas.save(); 599 | 600 | // compute the new extent and ready flag 601 | var extent = metric_.extent(); 602 | ready = extent.every(isFinite); 603 | if (extent_ != null) extent = extent_; 604 | 605 | // if this is an update (with no extent change), copy old values! 606 | var i0 = 0, max = Math.max(-extent[0], extent[1]); 607 | if (this === context) { 608 | if (max == max_) { 609 | i0 = width - cubism_metricOverlap; 610 | var dx = (start1 - start) / step; 611 | if (dx < width) { 612 | var canvas0 = buffer.getContext("2d"); 613 | canvas0.clearRect(0, 0, width, height); 614 | canvas0.drawImage(canvas.canvas, dx, 0, width - dx, height, 0, 0, width - dx, height); 615 | canvas.clearRect(0, 0, width, height); 616 | canvas.drawImage(canvas0.canvas, 0, 0); 617 | } 618 | } 619 | start = start1; 620 | } 621 | 622 | // update the domain 623 | scale.domain([0, max_ = max]); 624 | 625 | // clear for the new data 626 | canvas.clearRect(i0, 0, width - i0, height); 627 | 628 | // record whether there are negative values to display 629 | var negative; 630 | 631 | // positive bands 632 | for (var j = 0; j < m; ++j) { 633 | canvas.fillStyle = colors_[m + j]; 634 | 635 | // Adjust the range based on the current band index. 636 | var y0 = (j - m + 1) * height; 637 | scale.range([m * height + y0, y0]); 638 | y0 = scale(0); 639 | 640 | for (var i = i0, n = width, y1; i < n; ++i) { 641 | y1 = metric_.valueAt(i); 642 | if (y1 <= 0) { negative = true; continue; } 643 | if (y1 === undefined) continue; 644 | canvas.fillRect(i, y1 = scale(y1), 1, y0 - y1); 645 | } 646 | } 647 | 648 | if (negative) { 649 | // enable offset mode 650 | if (mode === "offset") { 651 | canvas.translate(0, height); 652 | canvas.scale(1, -1); 653 | } 654 | 655 | // negative bands 656 | for (var j = 0; j < m; ++j) { 657 | canvas.fillStyle = colors_[m - 1 - j]; 658 | 659 | // Adjust the range based on the current band index. 660 | var y0 = (j - m + 1) * height; 661 | scale.range([m * height + y0, y0]); 662 | y0 = scale(0); 663 | 664 | for (var i = i0, n = width, y1; i < n; ++i) { 665 | y1 = metric_.valueAt(i); 666 | if (y1 >= 0) continue; 667 | canvas.fillRect(i, scale(-y1), 1, y0 - scale(-y1)); 668 | } 669 | } 670 | } 671 | 672 | canvas.restore(); 673 | } 674 | 675 | function focus(i) { 676 | if (i == null) i = width - 1; 677 | var value = metric_.valueAt(i); 678 | span.datum(value).text(isNaN(value) ? null : format); 679 | } 680 | 681 | // Update the chart when the context changes. 682 | context.on("change.horizon-" + id, change); 683 | context.on("focus.horizon-" + id, focus); 684 | 685 | // Display the first metric change immediately, 686 | // but defer subsequent updates to the canvas change. 687 | // Note that someone still needs to listen to the metric, 688 | // so that it continues to update automatically. 689 | metric_.on("change.horizon-" + id, function(start, stop) { 690 | change(start, stop), focus(); 691 | if (ready) metric_.on("change.horizon-" + id, cubism_identity); 692 | }); 693 | }); 694 | } 695 | 696 | horizon.remove = function(selection) { 697 | 698 | selection 699 | .on("mousemove.horizon", null) 700 | .on("mouseout.horizon", null); 701 | 702 | selection.selectAll("canvas") 703 | .each(remove) 704 | .remove(); 705 | 706 | selection.selectAll(".title,.value") 707 | .remove(); 708 | 709 | function remove(d) { 710 | d.metric.on("change.horizon-" + d.id, null); 711 | context.on("change.horizon-" + d.id, null); 712 | context.on("focus.horizon-" + d.id, null); 713 | } 714 | }; 715 | 716 | horizon.mode = function(_) { 717 | if (!arguments.length) return mode; 718 | mode = _ + ""; 719 | return horizon; 720 | }; 721 | 722 | horizon.height = function(_) { 723 | if (!arguments.length) return height; 724 | buffer.height = height = +_; 725 | return horizon; 726 | }; 727 | 728 | horizon.metric = function(_) { 729 | if (!arguments.length) return metric; 730 | metric = _; 731 | return horizon; 732 | }; 733 | 734 | horizon.scale = function(_) { 735 | if (!arguments.length) return scale; 736 | scale = _; 737 | return horizon; 738 | }; 739 | 740 | horizon.extent = function(_) { 741 | if (!arguments.length) return extent; 742 | extent = _; 743 | return horizon; 744 | }; 745 | 746 | horizon.title = function(_) { 747 | if (!arguments.length) return title; 748 | title = _; 749 | return horizon; 750 | }; 751 | 752 | horizon.format = function(_) { 753 | if (!arguments.length) return format; 754 | format = _; 755 | return horizon; 756 | }; 757 | 758 | horizon.colors = function(_) { 759 | if (!arguments.length) return colors; 760 | colors = _; 761 | return horizon; 762 | }; 763 | 764 | return horizon; 765 | }; 766 | cubism_contextPrototype.comparison = function() { 767 | var context = this, 768 | width = context.size(), 769 | height = 120, 770 | scale = d3.scale.linear().interpolate(d3.interpolateRound), 771 | primary = function(d) { return d[0]; }, 772 | secondary = function(d) { return d[1]; }, 773 | extent = null, 774 | title = cubism_identity, 775 | formatPrimary = cubism_comparisonPrimaryFormat, 776 | formatChange = cubism_comparisonChangeFormat, 777 | colors = ["#9ecae1", "#225b84", "#a1d99b", "#22723a"], 778 | strokeWidth = 1.5; 779 | 780 | function comparison(selection) { 781 | 782 | selection 783 | .on("mousemove.comparison", function() { context.focus(Math.round(d3.mouse(this)[0])); }) 784 | .on("mouseout.comparison", function() { context.focus(null); }); 785 | 786 | selection.append("canvas") 787 | .attr("width", width) 788 | .attr("height", height); 789 | 790 | selection.append("span") 791 | .attr("class", "title") 792 | .text(title); 793 | 794 | selection.append("span") 795 | .attr("class", "value primary"); 796 | 797 | selection.append("span") 798 | .attr("class", "value change"); 799 | 800 | selection.each(function(d, i) { 801 | var that = this, 802 | id = ++cubism_id, 803 | primary_ = typeof primary === "function" ? primary.call(that, d, i) : primary, 804 | secondary_ = typeof secondary === "function" ? secondary.call(that, d, i) : secondary, 805 | extent_ = typeof extent === "function" ? extent.call(that, d, i) : extent, 806 | div = d3.select(that), 807 | canvas = div.select("canvas"), 808 | spanPrimary = div.select(".value.primary"), 809 | spanChange = div.select(".value.change"), 810 | ready; 811 | 812 | canvas.datum({id: id, primary: primary_, secondary: secondary_}); 813 | canvas = canvas.node().getContext("2d"); 814 | 815 | function change(start, stop) { 816 | canvas.save(); 817 | canvas.clearRect(0, 0, width, height); 818 | 819 | // update the scale 820 | var primaryExtent = primary_.extent(), 821 | secondaryExtent = secondary_.extent(), 822 | extent = extent_ == null ? primaryExtent : extent_; 823 | scale.domain(extent).range([height, 0]); 824 | ready = primaryExtent.concat(secondaryExtent).every(isFinite); 825 | 826 | // consistent overplotting 827 | var round = start / context.step() & 1 828 | ? cubism_comparisonRoundOdd 829 | : cubism_comparisonRoundEven; 830 | 831 | // positive changes 832 | canvas.fillStyle = colors[2]; 833 | for (var i = 0, n = width; i < n; ++i) { 834 | var y0 = scale(primary_.valueAt(i)), 835 | y1 = scale(secondary_.valueAt(i)); 836 | if (y0 < y1) canvas.fillRect(round(i), y0, 1, y1 - y0); 837 | } 838 | 839 | // negative changes 840 | canvas.fillStyle = colors[0]; 841 | for (i = 0; i < n; ++i) { 842 | var y0 = scale(primary_.valueAt(i)), 843 | y1 = scale(secondary_.valueAt(i)); 844 | if (y0 > y1) canvas.fillRect(round(i), y1, 1, y0 - y1); 845 | } 846 | 847 | // positive values 848 | canvas.fillStyle = colors[3]; 849 | for (i = 0; i < n; ++i) { 850 | var y0 = scale(primary_.valueAt(i)), 851 | y1 = scale(secondary_.valueAt(i)); 852 | if (y0 <= y1) canvas.fillRect(round(i), y0, 1, strokeWidth); 853 | } 854 | 855 | // negative values 856 | canvas.fillStyle = colors[1]; 857 | for (i = 0; i < n; ++i) { 858 | var y0 = scale(primary_.valueAt(i)), 859 | y1 = scale(secondary_.valueAt(i)); 860 | if (y0 > y1) canvas.fillRect(round(i), y0 - strokeWidth, 1, strokeWidth); 861 | } 862 | 863 | canvas.restore(); 864 | } 865 | 866 | function focus(i) { 867 | if (i == null) i = width - 1; 868 | var valuePrimary = primary_.valueAt(i), 869 | valueSecondary = secondary_.valueAt(i), 870 | valueChange = (valuePrimary - valueSecondary) / valueSecondary; 871 | 872 | spanPrimary 873 | .datum(valuePrimary) 874 | .text(isNaN(valuePrimary) ? null : formatPrimary); 875 | 876 | spanChange 877 | .datum(valueChange) 878 | .text(isNaN(valueChange) ? null : formatChange) 879 | .attr("class", "value change " + (valueChange > 0 ? "positive" : valueChange < 0 ? "negative" : "")); 880 | } 881 | 882 | // Display the first primary change immediately, 883 | // but defer subsequent updates to the context change. 884 | // Note that someone still needs to listen to the metric, 885 | // so that it continues to update automatically. 886 | primary_.on("change.comparison-" + id, firstChange); 887 | secondary_.on("change.comparison-" + id, firstChange); 888 | function firstChange(start, stop) { 889 | change(start, stop), focus(); 890 | if (ready) { 891 | primary_.on("change.comparison-" + id, cubism_identity); 892 | secondary_.on("change.comparison-" + id, cubism_identity); 893 | } 894 | } 895 | 896 | // Update the chart when the context changes. 897 | context.on("change.comparison-" + id, change); 898 | context.on("focus.comparison-" + id, focus); 899 | }); 900 | } 901 | 902 | comparison.remove = function(selection) { 903 | 904 | selection 905 | .on("mousemove.comparison", null) 906 | .on("mouseout.comparison", null); 907 | 908 | selection.selectAll("canvas") 909 | .each(remove) 910 | .remove(); 911 | 912 | selection.selectAll(".title,.value") 913 | .remove(); 914 | 915 | function remove(d) { 916 | d.primary.on("change.comparison-" + d.id, null); 917 | d.secondary.on("change.comparison-" + d.id, null); 918 | context.on("change.comparison-" + d.id, null); 919 | context.on("focus.comparison-" + d.id, null); 920 | } 921 | }; 922 | 923 | comparison.height = function(_) { 924 | if (!arguments.length) return height; 925 | height = +_; 926 | return comparison; 927 | }; 928 | 929 | comparison.primary = function(_) { 930 | if (!arguments.length) return primary; 931 | primary = _; 932 | return comparison; 933 | }; 934 | 935 | comparison.secondary = function(_) { 936 | if (!arguments.length) return secondary; 937 | secondary = _; 938 | return comparison; 939 | }; 940 | 941 | comparison.scale = function(_) { 942 | if (!arguments.length) return scale; 943 | scale = _; 944 | return comparison; 945 | }; 946 | 947 | comparison.extent = function(_) { 948 | if (!arguments.length) return extent; 949 | extent = _; 950 | return comparison; 951 | }; 952 | 953 | comparison.title = function(_) { 954 | if (!arguments.length) return title; 955 | title = _; 956 | return comparison; 957 | }; 958 | 959 | comparison.formatPrimary = function(_) { 960 | if (!arguments.length) return formatPrimary; 961 | formatPrimary = _; 962 | return comparison; 963 | }; 964 | 965 | comparison.formatChange = function(_) { 966 | if (!arguments.length) return formatChange; 967 | formatChange = _; 968 | return comparison; 969 | }; 970 | 971 | comparison.colors = function(_) { 972 | if (!arguments.length) return colors; 973 | colors = _; 974 | return comparison; 975 | }; 976 | 977 | comparison.strokeWidth = function(_) { 978 | if (!arguments.length) return strokeWidth; 979 | strokeWidth = _; 980 | return comparison; 981 | }; 982 | 983 | return comparison; 984 | }; 985 | 986 | var cubism_comparisonPrimaryFormat = d3.format(".2s"), 987 | cubism_comparisonChangeFormat = d3.format("+.0%"); 988 | 989 | function cubism_comparisonRoundEven(i) { 990 | return i & 0xfffffe; 991 | } 992 | 993 | function cubism_comparisonRoundOdd(i) { 994 | return ((i + 1) & 0xfffffe) - 1; 995 | } 996 | cubism_contextPrototype.axis = function() { 997 | var context = this, 998 | scale = context.scale, 999 | axis_ = d3.svg.axis().scale(scale); 1000 | 1001 | var format = context.step() < 6e4 ? cubism_axisFormatSeconds 1002 | : context.step() < 864e5 ? cubism_axisFormatMinutes 1003 | : cubism_axisFormatDays; 1004 | 1005 | function axis(selection) { 1006 | var id = ++cubism_id, 1007 | tick; 1008 | 1009 | var g = selection.append("svg") 1010 | .datum({id: id}) 1011 | .attr("width", context.size()) 1012 | .attr("height", Math.max(28, -axis.tickSize())) 1013 | .append("g") 1014 | .attr("transform", "translate(0," + (axis_.orient() === "top" ? 27 : 4) + ")") 1015 | .call(axis_); 1016 | 1017 | context.on("change.axis-" + id, function() { 1018 | g.call(axis_); 1019 | if (!tick) tick = d3.select(g.node().appendChild(g.selectAll("text").node().cloneNode(true))) 1020 | .style("display", "none") 1021 | .text(null); 1022 | }); 1023 | 1024 | context.on("focus.axis-" + id, function(i) { 1025 | if (tick) { 1026 | if (i == null) { 1027 | tick.style("display", "none"); 1028 | g.selectAll("text").style("fill-opacity", null); 1029 | } else { 1030 | tick.style("display", null).attr("x", i).text(format(scale.invert(i))); 1031 | var dx = tick.node().getComputedTextLength() + 6; 1032 | g.selectAll("text").style("fill-opacity", function(d) { return Math.abs(scale(d) - i) < dx ? 0 : 1; }); 1033 | } 1034 | } 1035 | }); 1036 | } 1037 | 1038 | axis.remove = function(selection) { 1039 | 1040 | selection.selectAll("svg") 1041 | .each(remove) 1042 | .remove(); 1043 | 1044 | function remove(d) { 1045 | context.on("change.axis-" + d.id, null); 1046 | context.on("focus.axis-" + d.id, null); 1047 | } 1048 | }; 1049 | 1050 | return d3.rebind(axis, axis_, 1051 | "orient", 1052 | "ticks", 1053 | "tickSubdivide", 1054 | "tickSize", 1055 | "tickPadding", 1056 | "tickFormat"); 1057 | }; 1058 | 1059 | var cubism_axisFormatSeconds = d3.time.format("%I:%M:%S %p"), 1060 | cubism_axisFormatMinutes = d3.time.format("%I:%M %p"), 1061 | cubism_axisFormatDays = d3.time.format("%B %d"); 1062 | cubism_contextPrototype.rule = function() { 1063 | var context = this, 1064 | metric = cubism_identity; 1065 | 1066 | function rule(selection) { 1067 | var id = ++cubism_id; 1068 | 1069 | var line = selection.append("div") 1070 | .datum({id: id}) 1071 | .attr("class", "line") 1072 | .call(cubism_ruleStyle); 1073 | 1074 | selection.each(function(d, i) { 1075 | var that = this, 1076 | id = ++cubism_id, 1077 | metric_ = typeof metric === "function" ? metric.call(that, d, i) : metric; 1078 | 1079 | if (!metric_) return; 1080 | 1081 | function change(start, stop) { 1082 | var values = []; 1083 | 1084 | for (var i = 0, n = context.size(); i < n; ++i) { 1085 | if (metric_.valueAt(i)) { 1086 | values.push(i); 1087 | } 1088 | } 1089 | 1090 | var lines = selection.selectAll(".metric").data(values); 1091 | lines.exit().remove(); 1092 | lines.enter().append("div").attr("class", "metric line").call(cubism_ruleStyle); 1093 | lines.style("left", cubism_ruleLeft); 1094 | } 1095 | 1096 | context.on("change.rule-" + id, change); 1097 | metric_.on("change.rule-" + id, change); 1098 | }); 1099 | 1100 | context.on("focus.rule-" + id, function(i) { 1101 | line.datum(i) 1102 | .style("display", i == null ? "none" : null) 1103 | .style("left", i == null ? null : cubism_ruleLeft); 1104 | }); 1105 | } 1106 | 1107 | rule.remove = function(selection) { 1108 | 1109 | selection.selectAll(".line") 1110 | .each(remove) 1111 | .remove(); 1112 | 1113 | function remove(d) { 1114 | context.on("focus.rule-" + d.id, null); 1115 | } 1116 | }; 1117 | 1118 | rule.metric = function(_) { 1119 | if (!arguments.length) return metric; 1120 | metric = _; 1121 | return rule; 1122 | }; 1123 | 1124 | return rule; 1125 | }; 1126 | 1127 | function cubism_ruleStyle(line) { 1128 | line 1129 | .style("position", "absolute") 1130 | .style("top", 0) 1131 | .style("bottom", 0) 1132 | .style("width", "1px") 1133 | .style("pointer-events", "none"); 1134 | } 1135 | 1136 | function cubism_ruleLeft(i) { 1137 | return i + "px"; 1138 | } 1139 | })(this); 1140 | --------------------------------------------------------------------------------