├── .npmignore ├── README.md ├── .gitignore ├── definitions ├── Collect.js ├── Extent.js ├── Filter.js ├── Sample.js ├── Formula.js ├── Cross.js ├── Rank.js ├── Fold.js ├── Range.js ├── Impute.js ├── Lookup.js ├── CountPattern.js ├── Aggregate.js └── Bin.js ├── .eslintrc ├── src ├── transforms │ ├── Key.js │ ├── MultiValues.js │ ├── Params.js │ ├── Sieve.js │ ├── Compare.js │ ├── Field.js │ ├── MultiExtent.js │ ├── Range.js │ ├── Formula.js │ ├── Extent.js │ ├── Values.js │ ├── TupleIndex.js │ ├── Relay.js │ ├── PreFacet.js │ ├── Filter.js │ ├── Generate.js │ ├── Collect.js │ ├── Bin.js │ ├── Cross.js │ ├── Rank.js │ ├── Lookup.js │ ├── Subflow.js │ ├── Fold.js │ ├── Sample.js │ ├── aggregate │ │ ├── TupleStore.js │ │ ├── Measures.js │ │ └── Aggregate.js │ ├── CountPattern.js │ ├── Facet.js │ └── Impute.js ├── util │ ├── UniqueList.js │ └── Heap.js ├── register.js ├── dataflow │ ├── connect.js │ ├── rank.js │ ├── events.js │ ├── load.js │ ├── add.js │ ├── update.js │ ├── on.js │ └── run.js ├── Transform.js ├── Tuple.js ├── ChangeSet.js ├── Parameters.js ├── MultiPulse.js ├── EventStream.js ├── Dataflow.js ├── Operator.js └── Pulse.js ├── test ├── transforms │ ├── multiextent-test.js │ ├── extent-test.js │ ├── compare-test.js │ ├── field-test.js │ ├── rank-test.js │ ├── impute-test.js │ ├── formula-test.js │ ├── fold-test.js │ ├── lookup-test.js │ ├── relay-test.js │ ├── collect-test.js │ ├── sample-test.js │ ├── tupleindex-test.js │ ├── prefacet-test.js │ ├── values-test.js │ ├── aggregate-test.js │ └── facet-test.js └── core │ ├── dataflow-test.js │ └── parameters-test.js ├── package.json ├── LICENSE └── index.js /.npmignore: -------------------------------------------------------------------------------- 1 | *.sublime-* 2 | build/*.zip 3 | test/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vega-dataflow 2 | 3 | Reactive dataflow processing. 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.sublime-workspace 2 | .DS_Store 3 | build/ 4 | node_modules 5 | npm-debug.log -------------------------------------------------------------------------------- /definitions/Collect.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Collect", 3 | "metadata": {"source": true}, 4 | "params": [ 5 | { "name": "sort", "type": "compare" } 6 | ] 7 | }; 8 | -------------------------------------------------------------------------------- /definitions/Extent.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Extent", 3 | "metadata": {}, 4 | "params": [ 5 | { "name": "field", "type": "field", "required": true } 6 | ] 7 | }; 8 | -------------------------------------------------------------------------------- /definitions/Filter.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Filter", 3 | "metadata": {"changes": true}, 4 | "params": [ 5 | { "name": "expr", "type": "expr", "required": true } 6 | ] 7 | }; 8 | -------------------------------------------------------------------------------- /definitions/Sample.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Sample", 3 | "metadata": {"source": true, "changes": true}, 4 | "params": [ 5 | { "name": "size", "type": "number", "default": 1000 } 6 | ] 7 | }; 8 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | parserOptions: 2 | sourceType: "module" 3 | 4 | env: 5 | es6: true 6 | browser: true 7 | node: true 8 | 9 | extends: 10 | "eslint:recommended" 11 | 12 | rules: 13 | no-cond-assign: 0 -------------------------------------------------------------------------------- /definitions/Formula.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Formula", 3 | "metadata": {"modifies": true}, 4 | "params": [ 5 | { "name": "expr", "type": "expr", "required": true }, 6 | { "name": "as", "type": "string", "required": true } 7 | ] 8 | }; 9 | -------------------------------------------------------------------------------- /definitions/Cross.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Cross", 3 | "metadata": {"source": true, "generates": true, "changes": true}, 4 | "params": [ 5 | { "name": "filter", "type": "expr" }, 6 | { "name": "as", "type": "string", "array": true, "length": 2, "default": ["a", "b"] } 7 | ] 8 | }; 9 | -------------------------------------------------------------------------------- /definitions/Rank.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Rank", 3 | "metadata": {"modifies": true}, 4 | "params": [ 5 | { "name": "field", "type": "field" }, 6 | { "name": "normalize", "type": "boolean", "default": false }, 7 | { "name": "as", "type": "string", "default": "rank" } 8 | ] 9 | }; 10 | -------------------------------------------------------------------------------- /definitions/Fold.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Fold", 3 | "metadata": {"generates": true, "changes": true}, 4 | "params": [ 5 | { "name": "fields", "type": "field", "array": true, "required": true }, 6 | { "name": "as", "type": "string", "array": true, "length": 2, "default": ["key", "value"] } 7 | ] 8 | }; 9 | -------------------------------------------------------------------------------- /definitions/Range.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Range", 3 | "metadata": {"generates": true, "source": true}, 4 | "params": [ 5 | { "name": "start", "type": "number", "required": true }, 6 | { "name": "stop", "type": "number", "required": true }, 7 | { "name": "step", "type": "number", "default": 1 } 8 | ], 9 | "output": ["value"] 10 | }; 11 | -------------------------------------------------------------------------------- /definitions/Impute.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Impute", 3 | "metadata": {"changes": true}, 4 | "params": [ 5 | { "name": "field", "type": "field", "required": true }, 6 | { "name": "groupby", "type": "field", "array": true }, 7 | { "name": "orderby", "type": "field", "array": true }, 8 | { "name": "method", "type": "enum", "default": "value", 9 | "values": ["value", "mean", "median", "max", "min"] }, 10 | { "name": "value", "default": 0 } 11 | ] 12 | }; 13 | -------------------------------------------------------------------------------- /definitions/Lookup.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Lookup", 3 | "metadata": {"modifies": true}, 4 | "params": [ 5 | { "name": "index", "type": "index", "params": [ 6 | {"name": "from", "type": "data", "required": true }, 7 | {"name": "key", "type": "field", "required": true } 8 | ] }, 9 | { "name": "fields", "type": "field", "array": true, "required": true }, 10 | { "name": "as", "type": "string", "array": true, "required": true }, 11 | { "name": "default", "default": null } 12 | ] 13 | }; 14 | -------------------------------------------------------------------------------- /definitions/CountPattern.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "CountPattern", 3 | "metadata": {"generates": true, "changes": true}, 4 | "params": [ 5 | { "name": "field", "type": "field", "required": true }, 6 | { "name": "case", "type": "enum", "values": ["upper", "lower", "mixed"], "default": "mixed" }, 7 | { "name": "pattern", "type": "string", "default": "[\\w\"]+" }, 8 | { "name": "stopwords", "type": "string", "default": "" }, 9 | { "name": "as", "type": "string", "array": true, "length": 2, "default": ["text", "count"] } 10 | ] 11 | }; 12 | -------------------------------------------------------------------------------- /src/transforms/Key.js: -------------------------------------------------------------------------------- 1 | import Operator from '../Operator'; 2 | import {inherits, key} from 'vega-util'; 3 | 4 | /** 5 | * Generates a key function. 6 | * @constructor 7 | * @param {object} params - The parameters for this operator. 8 | * @param {Array} params.fields - The field name(s) for the key function. 9 | */ 10 | export default function Key(params) { 11 | Operator.call(this, null, update, params); 12 | } 13 | 14 | inherits(Key, Operator); 15 | 16 | function update(_) { 17 | return (this.value && !_.modified()) ? this.value : key(_.fields); 18 | } 19 | -------------------------------------------------------------------------------- /test/transforms/multiextent-test.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape'), 2 | vega = require('../../'), 3 | MultiExtent = vega.transforms.MultiExtent; 4 | 5 | tape('MultiExtent combines extents', function(test) { 6 | var df = new vega.Dataflow(), 7 | e = df.add([10, 50]), 8 | m = df.add(MultiExtent, {extents: [ 9 | [-5, 0], [0, 20], e 10 | ]}); 11 | 12 | test.equal(m.value, null); 13 | 14 | df.run(); 15 | test.deepEqual(m.value, [-5, 50]); 16 | 17 | df.update(e, [0, 1]).run(); 18 | test.deepEqual(m.value, [-5, 20]); 19 | 20 | test.end(); 21 | }); 22 | -------------------------------------------------------------------------------- /src/util/UniqueList.js: -------------------------------------------------------------------------------- 1 | import {identity} from 'vega-util'; 2 | 3 | export default function UniqueList(idFunc) { 4 | var $ = idFunc || identity, 5 | list = [], 6 | ids = {}; 7 | 8 | list.add = function(_) { 9 | var id = $(_); 10 | if (!ids[id]) { 11 | ids[id] = 1; 12 | list.push(_); 13 | } 14 | return list; 15 | }; 16 | 17 | list.remove = function(_) { 18 | var id = $(_), idx; 19 | if (ids[id]) { 20 | ids[id] = 0; 21 | if ((idx = list.indexOf(_)) >= 0) { 22 | list.splice(idx, 1); 23 | } 24 | } 25 | return list; 26 | }; 27 | 28 | return list; 29 | } 30 | -------------------------------------------------------------------------------- /src/transforms/MultiValues.js: -------------------------------------------------------------------------------- 1 | import Operator from '../Operator'; 2 | import {inherits} from 'vega-util'; 3 | 4 | /** 5 | * Merge a collection of value arrays. 6 | * @constructor 7 | * @param {object} params - The parameters for this operator. 8 | * @param {Array>} params.values - The input value arrrays. 9 | */ 10 | export default function MultiValues(params) { 11 | Operator.call(this, null, update, params); 12 | } 13 | 14 | inherits(MultiValues, Operator); 15 | 16 | function update(_) { 17 | return (this.value && !_.modified()) 18 | ? this.value 19 | : _.values.reduce(function(data, _) { return data.concat(_); }, []); 20 | } 21 | -------------------------------------------------------------------------------- /src/transforms/Params.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {inherits} from 'vega-util'; 3 | 4 | /** 5 | * Operator whose value is simply its parameter hash. This operator is 6 | * useful for enabling reactive updates to values of nested objects. 7 | * @constructor 8 | * @param {object} params - The parameters for this operator. 9 | */ 10 | export default function Params(params) { 11 | Transform.call(this, null, params); 12 | } 13 | 14 | inherits(Params, Transform); 15 | 16 | Params.prototype.transform = function(_, pulse) { 17 | this.modified(_.modified()); 18 | this.value = _; 19 | return pulse.fork(); // do not pass tuples 20 | }; 21 | -------------------------------------------------------------------------------- /src/register.js: -------------------------------------------------------------------------------- 1 | export var transforms = {}; 2 | 3 | export var definitions = {}; 4 | 5 | export function register(def, constructor) { 6 | var type = def.type; 7 | definition(type, def); 8 | transform(type, constructor); 9 | } 10 | 11 | export function definition(type, def) { 12 | type = type && type.toLowerCase(); 13 | return arguments.length > 1 ? (definitions[type] = def, this) 14 | : definitions.hasOwnProperty(type) ? definitions[type] : null; 15 | } 16 | 17 | export function transform(type, constructor) { 18 | return arguments.length > 1 ? (transforms[type] = constructor, this) 19 | : transforms.hasOwnProperty(type) ? transforms[type] : null; 20 | } 21 | -------------------------------------------------------------------------------- /src/dataflow/connect.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Connect a target operator as a dependent of source operators. 3 | * If necessary, this method will rerank the target operator and its 4 | * dependents to ensure propagation proceeds in a topologically sorted order. 5 | * @param {Operator} target - The target operator. 6 | * @param {Array} - The source operators that should propagate 7 | * to the target operator. 8 | */ 9 | export default function(target, sources) { 10 | var targetRank = target.rank, i, n; 11 | 12 | for (i=0, n=sources.length; i} params.fields - The fields to compare. 9 | * @param {Array} [params.orders] - The sort orders. 10 | * Each entry should be one of "ascending" (default) or "descending". 11 | */ 12 | export default function Compare(params) { 13 | Operator.call(this, null, update, params); 14 | } 15 | 16 | inherits(Compare, Operator); 17 | 18 | function update(_) { 19 | return (this.value && !_.modified()) 20 | ? this.value 21 | : compare(_.fields, _.orders); 22 | } 23 | -------------------------------------------------------------------------------- /definitions/Aggregate.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Aggregate", 3 | "metadata": {"generates": true, "changes": true}, 4 | "params": [ 5 | { "name": "groupby", "type": "field", "array": true }, 6 | { "name": "fields", "type": "field", "array": true }, 7 | { "name": "ops", "type": "enum", "array": true, 8 | "values": [ 9 | "count", "valid", "missing", "distinct", 10 | "sum", "mean", "average", "variance", "variancep", "stdev", 11 | "stdevp", "median", "q1", "q3", "modeskew", "min", "max", 12 | "argmin", "argmax" ] }, 13 | { "name": "as", "type": "string", "array": true }, 14 | { "name": "drop", "type": "boolean", "default": true }, 15 | { "name": "key", "type": "field" } 16 | ] 17 | }; 18 | -------------------------------------------------------------------------------- /src/dataflow/rank.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Assigns a rank to an operator. Ranks are assigned in increasing order 3 | * by incrementing an internal rank counter. 4 | * @param {Operator} op - The operator to assign a rank. 5 | */ 6 | export function rank(op) { 7 | op.rank = ++this._rank; 8 | } 9 | 10 | /** 11 | * Re-ranks an operator and all downstream target dependencies. This 12 | * is necessary when upstream depencies of higher rank are added to 13 | * a target operator. 14 | * @param {Operator} op - The operator to re-rank. 15 | */ 16 | export function rerank(op) { 17 | var queue = [op], 18 | cur, list, i; 19 | 20 | while (queue.length) { 21 | this.rank(cur = queue.pop()); 22 | if (list = cur._targets) { 23 | for (i=list.length; --i >= 0;) { 24 | queue.push(list[i]); 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /definitions/Bin.js: -------------------------------------------------------------------------------- 1 | export default { 2 | "type": "Bin", 3 | "metadata": {"modifies": true}, 4 | "params": [ 5 | { "name": "field", "type": "field", "required": true }, 6 | { "name": "maxbins", "type": "number", "default": 20 }, 7 | { "name": "base", "type": "number", "default": 10 }, 8 | { "name": "divide", "type": "number", "array": true, "default": [5, 2] }, 9 | { "name": "extent", "type": "number", "array": true, "length": 2, "required": true }, 10 | { "name": "step", "type": "number" }, 11 | { "name": "steps", "type": "number", "array": true }, 12 | { "name": "minstep", "type": "number", "default": 0 }, 13 | { "name": "nice", "type": "boolean", "default": true }, 14 | { "name": "name", "type": "string" }, 15 | { "name": "as", "type": "string", "array": true, "length": 2, "default": ["bin0", "bin1"] } 16 | ] 17 | }; -------------------------------------------------------------------------------- /src/transforms/Field.js: -------------------------------------------------------------------------------- 1 | import Operator from '../Operator'; 2 | import {array, field, inherits, isArray} from 'vega-util'; 3 | 4 | /** 5 | * Generates one or more field accessor functions. 6 | * If the 'name' parameter is an array, an array of field accessors 7 | * will be created and the 'as' parameter will be ignored. 8 | * @constructor 9 | * @param {object} params - The parameters for this operator. 10 | * @param {string} params.name - The field name(s) to access. 11 | * @param {string} params.as - The accessor function name. 12 | */ 13 | export default function Field(params) { 14 | Operator.call(this, null, update, params); 15 | } 16 | 17 | inherits(Field, Operator); 18 | 19 | function update(_) { 20 | return (this.value && !_.modified()) ? this.value 21 | : isArray(_.name) ? array(_.name).map(function(f) { return field(f); }) 22 | : field(_.name, _.as); 23 | } 24 | -------------------------------------------------------------------------------- /src/transforms/MultiExtent.js: -------------------------------------------------------------------------------- 1 | import Operator from '../Operator'; 2 | import {inherits} from 'vega-util'; 3 | 4 | /** 5 | * Computes global min/max extents over a collection of extents. 6 | * @constructor 7 | * @param {object} params - The parameters for this operator. 8 | * @param {Array>} params.extents - The input extents. 9 | */ 10 | export default function MultiExtent(params) { 11 | Operator.call(this, null, update, params); 12 | } 13 | 14 | inherits(MultiExtent, Operator); 15 | 16 | function update(_) { 17 | if (this.value && !_.modified()) { 18 | return this.value; 19 | } 20 | 21 | var min = +Infinity, 22 | max = -Infinity, 23 | ext = _.extents, 24 | i, n, e; 25 | 26 | for (i=0, n=ext.length; i max) max = e[1]; 30 | } 31 | return [min, max]; 32 | } 33 | -------------------------------------------------------------------------------- /test/transforms/extent-test.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape'), 2 | util = require('vega-util'), 3 | vega = require('../../'), 4 | changeset = vega.changeset, 5 | Collect = vega.transforms.Collect, 6 | Extent = vega.transforms.Extent; 7 | 8 | tape('Extent computes extents', function(test) { 9 | var data = [ 10 | {"x": 0, "y": 28}, {"x": 1, "y": 43}, 11 | {"x": 0, "y": 55}, {"x": 1, "y": 72} 12 | ]; 13 | 14 | var x = util.field('x'), 15 | y = util.field('y'), 16 | df = new vega.Dataflow(), 17 | f = df.add(null), 18 | c = df.add(Collect), 19 | a = df.add(Extent, {field:f, pulse:c}), 20 | b = df.add(Extent, {field:y, pulse:c}); 21 | 22 | df.update(f, x) 23 | .pulse(c, changeset().insert(data)) 24 | .run(); 25 | test.deepEqual(a.value, [0, 1]); 26 | test.deepEqual(b.value, [28, 72]); 27 | 28 | df.update(f, y).run(); 29 | test.deepEqual(a.value, [28, 72]); 30 | test.deepEqual(b.value, [28, 72]); 31 | 32 | test.end(); 33 | }); 34 | -------------------------------------------------------------------------------- /src/transforms/Range.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {ingest} from '../Tuple'; 3 | import {inherits} from 'vega-util'; 4 | import {range} from 'd3-array'; 5 | 6 | /** 7 | * Generates data tuples for a specified range of numbers. 8 | * @constructor 9 | * @param {object} params - The parameters for this operator. 10 | * @param {number} params.start - The first number in the range. 11 | * @param {number} params.stop - The last number (exclusive) in the range. 12 | * @param {number} [params.step=1] - The step size between numbers in the range. 13 | */ 14 | export default function Range(params) { 15 | Transform.call(this, [], params); 16 | } 17 | 18 | var prototype = inherits(Range, Transform); 19 | 20 | prototype.transform = function(_, pulse) { 21 | if (!_.modified()) return; 22 | 23 | var out = pulse.materialize().fork(pulse.MOD); 24 | 25 | out.rem = pulse.rem.concat(this.value); 26 | out.source = this.value = range(_.start, _.stop, _.step).map(ingest); 27 | out.add = pulse.add.concat(this.value); 28 | 29 | return out; 30 | }; 31 | -------------------------------------------------------------------------------- /test/transforms/compare-test.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape'), 2 | vega = require('../../'), 3 | Compare = vega.transforms.Compare; 4 | 5 | tape('Compare generates comparator functions', function(test) { 6 | var df = new vega.Dataflow(), 7 | c = df.add('foo'), 8 | o = df.add('ascending'), 9 | f = df.add(Compare, {fields:c, orders:o}); 10 | 11 | df.run(); 12 | test.equal(typeof f.value, 'function'); 13 | test.deepEqual(f.value.fields, ['foo']); 14 | 15 | df.update(o, 'descending').run(); 16 | test.equal(typeof f.value, 'function'); 17 | test.deepEqual(f.value.fields, ['foo']); 18 | 19 | df.update(c, 'bar').run(); 20 | test.equal(typeof f.value, 'function'); 21 | test.deepEqual(f.value.fields, ['bar']); 22 | 23 | df.update(c, ['foo', 'bar']) 24 | .update(o, ['descending', 'descending']) 25 | .run(); 26 | test.equal(typeof f.value, 'function'); 27 | test.deepEqual(f.value.fields, ['foo', 'bar']); 28 | 29 | df.update(c, null).update(o, null).run(); 30 | test.equal(f.value, null); 31 | 32 | test.end(); 33 | }); 34 | -------------------------------------------------------------------------------- /src/transforms/Formula.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {inherits} from 'vega-util'; 3 | 4 | /** 5 | * Invokes a function for each data tuple and saves the results as a new field. 6 | * @constructor 7 | * @param {object} params - The parameters for this operator. 8 | * @param {function(object): *} params.expr - The formula function to invoke for each tuple. 9 | * @param {string} params.as - The field name under which to save the result. 10 | */ 11 | export default function Formula(params) { 12 | Transform.call(this, null, params); 13 | } 14 | 15 | var prototype = inherits(Formula, Transform); 16 | 17 | prototype.transform = function(_, pulse) { 18 | var func = _.expr, 19 | as = _.as, 20 | mod; 21 | 22 | function set(t) { 23 | t[as] = func(t, _); 24 | } 25 | 26 | if (_.modified()) { 27 | // parameters updated, need to reflow 28 | pulse.materialize().reflow().visit(pulse.SOURCE, set); 29 | } else { 30 | mod = pulse.modified(func.fields); 31 | pulse.visit(mod ? pulse.ADD_MOD : pulse.ADD, set); 32 | } 33 | 34 | return pulse.modifies(as); 35 | }; 36 | -------------------------------------------------------------------------------- /test/core/dataflow-test.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape'), 2 | vega = require('../../'); 3 | 4 | tape('Dataflow propagates values', function(test) { 5 | var df = new vega.Dataflow(), 6 | s1 = df.add(10), 7 | s2 = df.add(3), 8 | n1 = df.add(function(_) { return _.s1 + 0.25; }, {s1:s1}), 9 | n2 = df.add(function(_) { return _.n1 * _.s2; }, {n1:n1, s2:s2}), 10 | op = [s1, s2, n1, n2], 11 | stamp = function(_) { return _.stamp; }; 12 | 13 | test.equal(df.stamp(), 0); // timestamp 0 14 | 15 | test.equal(df.run(), 4); // run 4 ops 16 | test.equal(df.stamp(), 1); // timestamp 1 17 | test.deepEqual(op.map(stamp), [1, 1, 1, 1]); 18 | test.equal(n2.value, 30.75); 19 | 20 | test.equal(df.update(s1, 5).run(), 3); // run 3 ops 21 | test.equal(df.stamp(), 2); // timestamp 2 22 | test.deepEqual(op.map(stamp), [2, 1, 2, 2]); 23 | test.equal(n2.value, 15.75); 24 | 25 | test.equal(df.update(s2, 1).run(), 2); // run 2 ops 26 | test.equal(df.stamp(), 3); // timestamp 3 27 | test.deepEqual(op.map(stamp), [2, 3, 2, 3]); 28 | test.equal(n2.value, 5.25); 29 | 30 | test.end(); 31 | }); 32 | -------------------------------------------------------------------------------- /src/transforms/Extent.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {inherits} from 'vega-util'; 3 | 4 | /** 5 | * Computes extents (min/max) for a data field. 6 | * @constructor 7 | * @param {object} params - The parameters for this operator. 8 | * @param {function(object): *} params.field - The field over which to compute extends. 9 | */ 10 | export default function Extent(params) { 11 | Transform.call(this, [+Infinity, -Infinity], params); 12 | } 13 | 14 | var prototype = inherits(Extent, Transform); 15 | 16 | prototype.transform = function(_, pulse) { 17 | var extent = this.value, 18 | field = _.field, 19 | min = extent[0], 20 | max = extent[1], 21 | flag = pulse.ADD, 22 | mod; 23 | 24 | mod = pulse.changed() 25 | || pulse.modified(field.fields) 26 | || _.modified('field'); 27 | 28 | if (mod) { 29 | flag = pulse.SOURCE; 30 | min = +Infinity; 31 | max = -Infinity; 32 | } 33 | 34 | pulse.visit(flag, function(t) { 35 | var v = field(t); 36 | if (v < min) min = v; 37 | if (v > max) max = v; 38 | }); 39 | 40 | this.value = [min, max]; 41 | }; 42 | -------------------------------------------------------------------------------- /src/transforms/Values.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {inherits} from 'vega-util'; 3 | 4 | /** 5 | * Extracts an array of values. Assumes the source data has already been 6 | * reduced as needed (e.g., by an upstream Aggregate transform). 7 | * @constructor 8 | * @param {object} params - The parameters for this operator. 9 | * @param {function(object): *} params.field - The domain field to extract. 10 | * @param {function(*,*): number} [params.sort] - An optional 11 | * comparator function for sorting the values. The comparator will be 12 | * applied to backing tuples prior to value extraction. 13 | */ 14 | export default function Values(params) { 15 | Transform.call(this, null, params); 16 | } 17 | 18 | var prototype = inherits(Values, Transform); 19 | 20 | prototype.transform = function(_, pulse) { 21 | var run = !this.value 22 | || _.modified('field') 23 | || _.modified('sort') 24 | || pulse.changed() 25 | || (_.sort && pulse.modified(_.sort.fields)); 26 | 27 | if (run) { 28 | this.value = (_.sort 29 | ? pulse.source.slice().sort(_.sort) 30 | : pulse.source).map(_.field); 31 | } 32 | }; 33 | -------------------------------------------------------------------------------- /src/transforms/TupleIndex.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {inherits} from 'vega-util'; 3 | 4 | /** 5 | * An index that maps from unique, string-coerced, field values to tuples. 6 | * Assumes that the field serves as a unique key with no duplicate values. 7 | * @constructor 8 | * @param {object} params - The parameters for this operator. 9 | * @param {function(object): *} params.field - The field accessor to index. 10 | */ 11 | export default function TupleIndex(params) { 12 | Transform.call(this, {}, params); 13 | } 14 | 15 | var prototype = inherits(TupleIndex, Transform); 16 | 17 | prototype.transform = function(_, pulse) { 18 | var field = _.field, 19 | index = this.value, 20 | mod = true; 21 | 22 | function set(t) { index[field(t)] = t; } 23 | 24 | if (_.modified('field') || pulse.modified(field.fields)) { 25 | this.value = index = {}; 26 | pulse.visit(pulse.SOURCE, set); 27 | } else if (pulse.changed()) { 28 | pulse.visit(pulse.REM, function(t) { index[field(t)] = undefined; }); 29 | pulse.visit(pulse.ADD, set); 30 | } else { 31 | mod = false; 32 | } 33 | 34 | this.modified(mod); 35 | return pulse.fork(); 36 | }; 37 | -------------------------------------------------------------------------------- /src/dataflow/events.js: -------------------------------------------------------------------------------- 1 | import {stream} from '../EventStream'; 2 | import {array} from 'vega-util'; 3 | 4 | /** 5 | * Create a new event stream from an event source. 6 | * @param {object} source - The event source to monitor. The input must 7 | * support the addEventListener method. 8 | * @param {string} type - The event type. 9 | * @param {function(object): boolean} [filter] - Event filter function. 10 | * @param {function(object): *} [apply] - Event application function. 11 | * If provided, this function will be invoked and the result will be 12 | * used as the downstream event value. 13 | * @return {EventStream} 14 | */ 15 | export default function(source, type, filter, apply) { 16 | var df = this, 17 | s = stream(filter, apply), 18 | send = function(e) { 19 | e.dataflow = df; 20 | s.receive(e); 21 | df.run(); 22 | }, 23 | sources; 24 | 25 | if (typeof source === 'string' && typeof document !== 'undefined') { 26 | sources = document.querySelectorAll(source); 27 | } else { 28 | sources = array(source); 29 | } 30 | 31 | for (var i=0, n=sources.length; i} params.field - The field 12 | * accessor for an array of subflow tuple objects. 13 | */ 14 | export default function PreFacet(params) { 15 | Facet.call(this, params); 16 | } 17 | 18 | var prototype = inherits(PreFacet, Facet); 19 | 20 | prototype.transform = function(_, pulse) { 21 | var self = this, 22 | flow = _.subflow, 23 | field = _.field; 24 | 25 | if (_.modified('field') || field && pulse.modified(field.fields)) { 26 | error('PreFacet does not support field modification.'); 27 | } 28 | 29 | this._targets.active = 0; // reset list of active subflows 30 | 31 | pulse.visit(pulse.ADD, function(t) { 32 | var sf = self.subflow(t._id, flow, pulse, t); 33 | field ? field(t).forEach(function(_) { sf.add(ingest(_)); }) : sf.add(t); 34 | }); 35 | 36 | pulse.visit(pulse.REM, function(t) { 37 | var sf = self.subflow(t._id, flow, pulse, t); 38 | field ? field(t).forEach(function(_) { sf.rem(_); }) : sf.rem(t); 39 | }); 40 | 41 | return pulse; 42 | }; 43 | -------------------------------------------------------------------------------- /src/dataflow/add.js: -------------------------------------------------------------------------------- 1 | import Operator from '../Operator'; 2 | import {isFunction} from 'vega-util'; 3 | 4 | /** 5 | * Add an operator to the dataflow graph. This function accepts a 6 | * variety of input argument types. The basic signature support an 7 | * initial value, update function and parameters. If the first parameter 8 | * is an Operator instance, it will be added directly. If it is a 9 | * constructor for an Operator subclass, a new instance will be instantiated. 10 | * Otherwise, if the first parameter is a function instance, it will be used 11 | * as the update function and a null initial value is assumed. 12 | * @param {*} init - One of: the operator to add, the initial value of 13 | * the operator, an operator class to instantiate, or an update function. 14 | * @param {function} [update] - The operator update function. 15 | * @param {object} [params] - The operator parameters. 16 | * @param {boolean} [react=true] - Flag indicating if this operator should 17 | * listen for changes to upstream operators included as parameters. 18 | * @return {Operator} - The added operator. 19 | */ 20 | export default function(init, update, params, react) { 21 | var shift = 1, 22 | op = (init instanceof Operator) ? init 23 | : init && init.prototype instanceof Operator ? new init() 24 | : isFunction(init) ? new Operator(null, init) 25 | : (shift = 0, new Operator(init, update)); 26 | 27 | this.rank(op); 28 | if (shift) react = params, params = update; 29 | if (params) this.connect(op, op.parameters(params, react)); 30 | this.touch(op); 31 | 32 | return op; 33 | } 34 | -------------------------------------------------------------------------------- /test/transforms/fold-test.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape'), 2 | util = require('vega-util'), 3 | vega = require('../../'), 4 | changeset = vega.changeset, 5 | Collect = vega.transforms.Collect, 6 | Fold = vega.transforms.Fold; 7 | 8 | tape('Fold folds tuples', function(test) { 9 | var data = [ 10 | {a:'!', b:5, c:7}, 11 | {a:'?', b:2, c:4} 12 | ]; 13 | 14 | var fields = ['b', 'c'].map(util.field), 15 | df = new vega.Dataflow(), 16 | c0 = df.add(Collect), 17 | fd = df.add(Fold, {fields: fields, pulse:c0}), 18 | out = df.add(Collect, {pulse:fd}), d; 19 | 20 | // -- process adds 21 | df.pulse(c0, changeset().insert(data)).run(); 22 | d = out.value; 23 | test.equal(d.length, 4); 24 | test.equal(d[0].key, 'b'); test.equal(d[0].value, 5); test.equal(d[0].a, '!'); 25 | test.equal(d[1].key, 'c'); test.equal(d[1].value, 7); test.equal(d[1].a, '!'); 26 | test.equal(d[2].key, 'b'); test.equal(d[2].value, 2); test.equal(d[2].a, '?'); 27 | test.equal(d[3].key, 'c'); test.equal(d[3].value, 4); test.equal(d[3].a, '?'); 28 | 29 | // -- process mods 30 | df.pulse(c0, changeset().modify(data[1], 'b', 9)).run(); 31 | d = out.value; 32 | test.equal(d[2].key, 'b'); test.equal(d[2].value, 9); test.equal(d[2].a, '?'); 33 | 34 | // -- process rems 35 | df.pulse(c0, changeset().remove(data[0])).run(); 36 | d = out.value; 37 | test.equal(d.length, 2); 38 | test.equal(d[0].key, 'b'); test.equal(d[0].value, 9); test.equal(d[0].a, '?'); 39 | test.equal(d[1].key, 'c'); test.equal(d[1].value, 4); test.equal(d[1].a, '?'); 40 | 41 | test.end(); 42 | }); 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, University of Washington Interactive Data Lab 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /src/transforms/Filter.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {inherits} from 'vega-util'; 3 | 4 | /** 5 | * Filters data tuples according to a predicate function. 6 | * @constructor 7 | * @param {object} params - The parameters for this operator. 8 | * @param {function(object): *} params.expr - The predicate expression function 9 | * that determines a tuple's filter status. Truthy values pass the filter. 10 | */ 11 | export default function Filter(params) { 12 | Transform.call(this, {}, params); 13 | } 14 | 15 | var prototype = inherits(Filter, Transform); 16 | 17 | prototype.transform = function(_, pulse) { 18 | var test = _.expr, 19 | cache = this.value, // cache ids of filtered tuples 20 | output = pulse.fork(), 21 | add = output.add, 22 | rem = output.rem, 23 | mod = output.mod, isMod = true; 24 | 25 | pulse.visit(pulse.REM, function(x) { 26 | if (!cache[x._id]) rem.push(x); 27 | else cache[x._id] = 0; 28 | }); 29 | 30 | pulse.visit(pulse.ADD, function(x) { 31 | if (test(x, _)) add.push(x); 32 | else cache[x._id] = 1; 33 | }); 34 | 35 | function revisit(x) { 36 | var b = test(x, _), 37 | s = cache[x._id]; 38 | if (b && s) { 39 | cache[x._id] = 0; 40 | add.push(x); 41 | } else if (!b && !s) { 42 | cache[x._id] = 1; 43 | rem.push(x); 44 | } else if (isMod && b && !s) { 45 | mod.push(x); 46 | } 47 | } 48 | 49 | pulse.visit(pulse.MOD, revisit); 50 | 51 | if (_.modified()) { 52 | isMod = false; 53 | pulse.visit(pulse.REFLOW, revisit); 54 | } 55 | 56 | return output; 57 | }; 58 | -------------------------------------------------------------------------------- /src/Transform.js: -------------------------------------------------------------------------------- 1 | import Operator from './Operator'; 2 | import {inherits} from 'vega-util'; 3 | 4 | /** 5 | * Abstract class for operators that process data tuples. 6 | * Subclasses must provide a {@link transform} method for operator processing. 7 | * @constructor 8 | * @param {*} [init] - The initial value for this operator. 9 | * @param {object} [params] - The parameters for this operator. 10 | * @param {Operator} [source] - The operator from which to receive pulses. 11 | */ 12 | export default function Transform(init, params) { 13 | Operator.call(this, init, null, params); 14 | } 15 | 16 | var prototype = inherits(Transform, Operator); 17 | 18 | /** 19 | * Overrides {@link Operator.evaluate} for transform operators. 20 | * Marshalls parameter values and then invokes {@link transform}. 21 | * @param {Pulse} pulse - the current dataflow pulse. 22 | * @return {Pulse} The output pulse (or StopPropagation). A falsy return 23 | value (including undefined) will let the input pulse pass through. 24 | */ 25 | prototype.evaluate = function(pulse) { 26 | var params = this.marshall(pulse.stamp), 27 | out = this.transform(params, pulse); 28 | params.clear(); 29 | return out; 30 | }; 31 | 32 | /** 33 | * Process incoming pulses. 34 | * Subclasses should override this method to implement transforms. 35 | * @param {Parameters} _ - The operator parameter values. 36 | * @param {Pulse} pulse - The current dataflow pulse. 37 | * @return {Pulse} The output pulse (or StopPropagation). A falsy return 38 | * value (including undefined) will let the input pulse pass through. 39 | */ 40 | prototype.transform = function() {}; 41 | -------------------------------------------------------------------------------- /src/transforms/Generate.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {ingest} from '../Tuple'; 3 | import {inherits} from 'vega-util'; 4 | 5 | /** 6 | * Generates data tuples using a provided generator function. 7 | * @constructor 8 | * @param {object} params - The parameters for this operator. 9 | * @param {function(Parameters): object} params.generator - A tuple generator 10 | * function. This function is given the operator parameters as input. 11 | * Changes to any additional parameters will not trigger re-calculation 12 | * of previously generated tuples. Only future tuples are affected. 13 | * @param {number} params.size - The number of tuples to produce. 14 | */ 15 | export default function Generate(params) { 16 | Transform.call(this, [], params); 17 | } 18 | 19 | var prototype = inherits(Generate, Transform); 20 | 21 | prototype.transform = function(_, pulse) { 22 | var data = this.value, 23 | out = pulse.fork(pulse.ALL), 24 | num = _.size - data.length, 25 | gen = _.generator, 26 | add, rem, t; 27 | 28 | if (num > 0) { 29 | // need more tuples, generate and add 30 | for (add=[]; --num >= 0;) { 31 | add.push(t = ingest(gen(_))); 32 | data.push(t); 33 | } 34 | out.add = out.add.length 35 | ? out.materialize(out.ADD).add.concat(add) 36 | : add; 37 | } else { 38 | // need fewer tuples, remove 39 | rem = data.slice(0, -num); 40 | out.rem = out.rem.length 41 | ? out.materialize(out.REM).rem.concat(rem) 42 | : rem; 43 | data = data.slice(-num); 44 | } 45 | 46 | out.source = this.value = data; 47 | return out; 48 | }; 49 | -------------------------------------------------------------------------------- /src/transforms/Collect.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {inherits, merge} from 'vega-util'; 3 | 4 | /** 5 | * Collects all data tuples that pass through this operator. 6 | * @constructor 7 | * @param {object} params - The parameters for this operator. 8 | * @param {function(*,*): number} [params.sort] - An optional 9 | * comparator function for additionally sorting the collected tuples. 10 | */ 11 | export default function Collect(params) { 12 | Transform.call(this, [], params); 13 | } 14 | 15 | var prototype = inherits(Collect, Transform); 16 | 17 | prototype.transform = function(_, pulse) { 18 | var out = pulse.fork(pulse.ALL), 19 | add = pulse.changed(pulse.ADD), 20 | mod = pulse.changed(), 21 | sort = _.sort, 22 | data = this.value, 23 | push = function(t) { data.push(t); }, 24 | n = 0, map; 25 | 26 | if (out.rem.length) { // build id map and filter data array 27 | map = {}; 28 | out.visit(out.REM, function(t) { map[t._id] = 1; ++n; }); 29 | data = data.filter(function(t) { return !map[t._id]; }); 30 | } 31 | 32 | if (sort) { 33 | // if sort criteria change, re-sort the full data array 34 | if (_.modified('sort') || pulse.modified(sort.fields)) { 35 | data.sort(sort); 36 | mod = true; 37 | } 38 | // if added tuples, sort them in place and then merge 39 | if (add) { 40 | data = merge(sort, data, out.add.sort(sort)); 41 | } 42 | } else if (add) { 43 | // no sort, so simply add new tuples 44 | out.visit(out.ADD, push); 45 | } 46 | 47 | this.modified(mod); 48 | this.value = out.source = data; 49 | return out; 50 | }; 51 | -------------------------------------------------------------------------------- /src/transforms/Bin.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {inherits, accessor, accessorFields, accessorName} from 'vega-util'; 3 | import {bin} from 'vega-statistics'; 4 | 5 | /** 6 | * Generates a binning function for discretizing data. 7 | * @constructor 8 | * @param {object} params - The parameters for this operator. The 9 | * provided values should be valid options for the {@link bin} function. 10 | * @param {function(object): *} params.field - The data field to bin. 11 | */ 12 | export default function Bin(params) { 13 | Transform.call(this, null, params); 14 | } 15 | 16 | var prototype = inherits(Bin, Transform); 17 | 18 | prototype.transform = function(_, pulse) { 19 | var bins = this._bins(_), 20 | step = bins.step, 21 | as = _.as || ['bin0', 'bin1'], 22 | b0 = as[0], 23 | b1 = as[1], 24 | flag = _.modified() ? (pulse.reflow(), pulse.SOURCE) 25 | : pulse.modified(accessorFields(_.field)) ? pulse.ADD_MOD 26 | : pulse.ADD; 27 | 28 | pulse.visit(flag, function(t) { 29 | t[b1] = (t[b0] = bins(t)) + step; 30 | }); 31 | 32 | return pulse.modifies(as); 33 | }; 34 | 35 | prototype._bins = function(_) { 36 | if (this.value && !_.modified()) { 37 | return this.value; 38 | } 39 | 40 | var field = _.field, 41 | bins = bin(_), 42 | start = bins.start, 43 | step = bins.step; 44 | 45 | var f = function(t) { 46 | var v = field(t); 47 | return v == null ? null 48 | : start + step * Math.floor((+v - start) / step); 49 | }; 50 | 51 | f.step = step; 52 | 53 | return this.value = accessor( 54 | f, 55 | accessorFields(field), 56 | _.name || 'bin_' + accessorName(field) 57 | ); 58 | }; 59 | -------------------------------------------------------------------------------- /src/transforms/Cross.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {ingest} from '../Tuple'; 3 | import {inherits, truthy} from 'vega-util'; 4 | 5 | /** 6 | * Perform a cross-product of a tuple stream with itself. 7 | * @constructor 8 | * @param {object} params - The parameters for this operator. 9 | * @param {function(object):boolean} [params.filter] - An optional filter 10 | * function for selectively including tuples in the cross product. 11 | * @param {Array} [params.as] - The names of the output fields. 12 | */ 13 | export default function Cross(params) { 14 | Transform.call(this, null, params); 15 | } 16 | 17 | var prototype = inherits(Cross, Transform); 18 | 19 | prototype.transform = function(_, pulse) { 20 | var out = pulse.fork(pulse.NO_SOURCE), 21 | data = this.value, 22 | as = _.as || ['a', 'b'], 23 | a = as[0], b = as[1], 24 | reset = !data 25 | || pulse.changed(pulse.ADD_REM) 26 | || _.modified('as') 27 | || _.modified('filter'); 28 | 29 | if (reset) { 30 | if (data) out.rem = data; 31 | out.add = this.value = cross(pulse.source, a, b, _.filter || truthy); 32 | } else { 33 | out.mod = data; 34 | } 35 | 36 | return out.source = this.value, out.modifies(as); 37 | }; 38 | 39 | function cross(input, a, b, filter) { 40 | var data = [], 41 | t = {}, 42 | n = input.length, 43 | i = 0, 44 | j, left; 45 | 46 | for (; i} params.as - Output field names for each lookup value. 11 | * @param {*} [params.default] - A default value to use if lookup fails. 12 | */ 13 | export default function Lookup(params) { 14 | Transform.call(this, {}, params); 15 | } 16 | 17 | var prototype = inherits(Lookup, Transform); 18 | 19 | function get(index, key) { 20 | return index.hasOwnProperty(key) ? index[key] : null; 21 | } 22 | 23 | prototype.transform = function(_, pulse) { 24 | var out = pulse, 25 | as = _.as, 26 | keys = _.fields, 27 | index = _.index, 28 | defaultValue = _.default==null ? null : _.default, 29 | reset = _.modified(), 30 | flag = pulse.ADD, 31 | set, key, field, mods; 32 | 33 | if (keys.length === 1) { 34 | key = keys[0]; 35 | field = as[0]; 36 | set = function(t) { 37 | var v = get(index, key(t)); 38 | t[field] = v==null ? defaultValue : v; 39 | }; 40 | } else { 41 | set = function(t) { 42 | for (var i=0, n=keys.length, v; i start) { 64 | pidx = (idx - 1) >> 1; 65 | parent = array[pidx]; 66 | if (cmp(item, parent) < 0) { 67 | array[idx] = parent; 68 | idx = pidx; 69 | continue; 70 | } 71 | break; 72 | } 73 | return (array[idx] = item); 74 | } 75 | 76 | function siftup(array, idx, cmp) { 77 | var start = idx, 78 | end = array.length, 79 | item = array[idx], 80 | cidx = 2 * idx + 1, ridx; 81 | 82 | while (cidx < end) { 83 | ridx = cidx + 1; 84 | if (ridx < end && cmp(array[cidx], array[ridx]) >= 0) { 85 | cidx = ridx; 86 | } 87 | array[idx] = array[cidx]; 88 | idx = cidx; 89 | cidx = 2 * idx + 1; 90 | } 91 | array[idx] = item; 92 | return siftdown(array, start, idx, cmp); 93 | } 94 | -------------------------------------------------------------------------------- /test/transforms/collect-test.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape'), 2 | util = require('vega-util'), 3 | vega = require('../../'), 4 | changeset = vega.changeset, 5 | Collect = vega.transforms.Collect; 6 | 7 | tape('Collect collects tuples', function(test) { 8 | var data = [ 9 | {'id': 1, 'value': 'foo'}, 10 | {'id': 3, 'value': 'bar'}, 11 | {'id': 5, 'value': 'baz'} 12 | ]; 13 | 14 | var df = new vega.Dataflow(), 15 | so = df.add(null), 16 | c0 = df.add(Collect, {sort:so}); 17 | 18 | df.run(); // initialize 19 | test.equal(c0.value.length, 0); 20 | test.equal(!!c0.modified(), false); 21 | 22 | // add data 23 | df.pulse(c0, changeset().insert(data)).run(); 24 | test.equal(c0.value.length, 3); 25 | test.equal(c0.value[0], data[0]); 26 | test.equal(c0.value[1], data[1]); 27 | test.equal(c0.value[2], data[2]); 28 | test.equal(!!c0.modified(), true); 29 | 30 | // sort data 31 | df.update(so, util.compare('value')).run(); 32 | test.equal(c0.value.length, 3); 33 | test.equal(c0.value[0], data[1]); 34 | test.equal(c0.value[1], data[2]); 35 | test.equal(c0.value[2], data[0]); 36 | test.equal(!!c0.modified(), true); 37 | 38 | // add new data 39 | data.push({id:2, value:'abc'}); 40 | df.pulse(c0, changeset().insert(data[3])).run(); 41 | test.equal(c0.value.length, 4); 42 | test.equal(c0.value[0], data[3]); 43 | test.equal(c0.value[1], data[1]); 44 | test.equal(c0.value[2], data[2]); 45 | test.equal(c0.value[3], data[0]); 46 | test.equal(!!c0.modified(), true); 47 | 48 | // remove data 49 | df.pulse(c0, changeset().remove(data[1])).run(); 50 | test.equal(c0.value.length, 3); 51 | test.equal(c0.value[0], data[3]); 52 | test.equal(c0.value[1], data[2]); 53 | test.equal(c0.value[2], data[0]); 54 | test.equal(!!c0.modified(), true); 55 | 56 | // modify data 57 | df.pulse(c0, changeset().modify(data[0], 'value', 'boo')).run(); 58 | test.equal(c0.value.length, 3); 59 | test.equal(c0.value[0], data[3]); 60 | test.equal(c0.value[1], data[2]); 61 | test.equal(c0.value[2], data[0]); 62 | test.equal(!!c0.modified(), true); 63 | 64 | // do nothing 65 | df.touch(c0).run(); 66 | test.equal(c0.value.length, 3); 67 | test.equal(!!c0.modified(), false); 68 | 69 | test.end(); 70 | }); 71 | -------------------------------------------------------------------------------- /src/Tuple.js: -------------------------------------------------------------------------------- 1 | var TUPLE_ID = 1; 2 | 3 | /** 4 | * Resets the internal tuple id counter to zero. 5 | */ 6 | function reset() { 7 | TUPLE_ID = 1; 8 | } 9 | 10 | /** 11 | * Returns the id of a tuple. 12 | * @param {Tuple} t - The input tuple. 13 | * @return the tuple id. 14 | */ 15 | function tupleid(t) { 16 | return t._id; 17 | } 18 | 19 | /** 20 | * Copy the values of one tuple to another (ignoring id and prev fields). 21 | * @param {Tuple} t - The tuple to copy from. 22 | * @param {Tuple} c - The tuple to write to. 23 | * @return The re-written tuple, same as the argument 'c'. 24 | */ 25 | function copy(t, c) { 26 | for (var k in t) { 27 | if (k !== '_id') c[k] = t[k]; 28 | } 29 | return c; 30 | } 31 | 32 | /** 33 | * Ingest an object or value as a data tuple. 34 | * If the input value is an object, an id field will be added to it. For 35 | * efficiency, the input object is modified directly. A copy is not made. 36 | * If the input value is a literal, it will be wrapped in a new object 37 | * instance, with the value accessible as the 'data' property. 38 | * @param datum - The value to ingest. 39 | * @return {Tuple} The ingested data tuple. 40 | */ 41 | function ingest(datum) { 42 | var tuple = (datum === Object(datum)) ? datum : {data: datum}; 43 | if (!tuple._id) tuple._id = ++TUPLE_ID; 44 | return tuple; 45 | } 46 | 47 | /** 48 | * Given a source tuple, return a derived copy. 49 | * @param {object} t - The source tuple. 50 | * @return {object} The derived tuple. 51 | */ 52 | function derive(t) { 53 | return ingest(copy(t, {})); 54 | } 55 | 56 | /** 57 | * Rederive a derived tuple by copying values from the source tuple. 58 | * @param {object} t - The source tuple. 59 | * @param {object} d - The derived tuple. 60 | * @return {object} The derived tuple. 61 | */ 62 | function rederive(t, d) { 63 | return copy(t, d); 64 | } 65 | 66 | /** 67 | * Replace an existing tuple with a new tuple. 68 | * The existing tuple will become the previous value of the new. 69 | * @param {object} t - The existing data tuple. 70 | * @param {object} d - The new tuple that replaces the old. 71 | * @return {object} The new tuple. 72 | */ 73 | function replace(t, d) { 74 | return d._id = t._id, d; 75 | } 76 | 77 | export { 78 | reset, tupleid, ingest, 79 | replace, derive, rederive 80 | }; 81 | -------------------------------------------------------------------------------- /src/transforms/Fold.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {derive, rederive} from '../Tuple'; 3 | import {inherits} from 'vega-util'; 4 | 5 | /** 6 | * Folds one more tuple fields into multiple tuples in which the field 7 | * name and values are available under new 'key' and 'value' fields. 8 | * @constructor 9 | * @param {object} params - The parameters for this operator. 10 | * @param {function(object): *} params.fields - An array of field accessors 11 | * for the tuple fields that should be folded. 12 | */ 13 | export default function Fold(params) { 14 | Transform.call(this, {}, params); 15 | } 16 | 17 | var prototype = inherits(Fold, Transform); 18 | 19 | function keyFunction(f) { 20 | return f.fields.join('|'); 21 | } 22 | 23 | prototype.transform = function(_, pulse) { 24 | var cache = this.value, 25 | reset = _.modified('fields'), 26 | fields = _.fields, 27 | as = _.as || ['key', 'value'], 28 | key = as[0], 29 | value = as[1], 30 | keys = fields.map(keyFunction), 31 | n = fields.length, 32 | stamp = pulse.stamp, 33 | out = pulse.fork(), 34 | i = 0, mask = 0, id; 35 | 36 | function add(t) { 37 | var f = (cache[t._id] = Array(n)); // create cache of folded tuples 38 | for (var i=0, ft; i= 0 ? index + ':' : '') + name; 17 | } 18 | 19 | /** 20 | * Set a parameter value. If the parameter value changes, the parameter 21 | * will be recorded as modified. 22 | * @param {string} name - The parameter name. 23 | * @param {number} index - The index into an array-value parameter. Ignored if 24 | * the argument is undefined, null or less than zero. 25 | * @param {*} value - The parameter value to set. 26 | * @param {boolean} [force=false] - If true, records the parameter as modified 27 | * even if the value is unchanged. 28 | * @return {Parameters} - This parameter object. 29 | */ 30 | prototype.set = function(name, index, value, force) { 31 | var o = this, 32 | v = o[name], 33 | mod = o[CACHE]; 34 | 35 | if (index != null && index >= 0) { 36 | if (v[index] !== value || force) { 37 | v[index] = value; 38 | mod[key(name, index)] = 1; 39 | mod[name] = 1; 40 | } 41 | } else if (v !== value || force) { 42 | o[name] = value; 43 | mod[name] = 1; 44 | if (isArray(value)) value.forEach(function(v, i) { 45 | mod[key(name, i)] = 1; 46 | }); 47 | } 48 | 49 | return o; 50 | }; 51 | 52 | /** 53 | * Tests if one or more parameters has been modified. If invoked with no 54 | * arguments, returns true if any parameter value has changed. If the first 55 | * argument is array, returns trues if any parameter name in the array has 56 | * changed. Otherwise, tests if the given name and optional array index has 57 | * changed. 58 | * @param {string} name - The parameter name to test. 59 | * @param {number} [index=undefined] - The parameter array index to test. 60 | * @return {boolean} - Returns true if a queried parameter was modified. 61 | */ 62 | prototype.modified = function(name, index) { 63 | var mod = this[CACHE], k; 64 | if (!arguments.length) { 65 | for (k in mod) { if (mod[k]) return true; } 66 | return false; 67 | } else if (isArray(name)) { 68 | for (k=0; k} pulses - The sub-pulses for this multi-pulse. 15 | */ 16 | export default function MultiPulse(dataflow, stamp, pulses, encode) { 17 | var p = this, 18 | c = 0, 19 | pulse, hash, i, n, f; 20 | 21 | this.dataflow = dataflow; 22 | this.stamp = stamp; 23 | this.fields = null; 24 | this.encode = encode || null; 25 | this.pulses = pulses; 26 | 27 | for (i=0, n=pulses.length; i= cap) { 36 | p = res[idx]; 37 | if (map[p._id]) out.rem.push(p); // eviction 38 | res[idx] = t; 39 | } 40 | } 41 | ++cnt; 42 | } 43 | 44 | if (pulse.rem.length) { 45 | // find all tuples that should be removed, add to output 46 | pulse.visit(pulse.REM, function(t) { 47 | if (map[t._id]) { 48 | map[t._id] = -1; 49 | out.rem.push(t); 50 | } 51 | --cnt; 52 | }); 53 | 54 | // filter removed tuples out of the sample reservoir 55 | res = res.filter(function(t) { return map[t._id] !== -1; }); 56 | } 57 | 58 | if ((pulse.rem.length || mod) && res.length < num && pulse.source) { 59 | // replenish sample if backing data source is available 60 | cap = cnt = res.length; 61 | pulse.visit(pulse.SOURCE, function(t) { 62 | // update, but skip previously sampled tuples 63 | if (!map[t._id]) update(t); 64 | }); 65 | cap = -1; 66 | } 67 | 68 | if (mod && res.length > num) { 69 | for (var i=0, n=res.length-num; i pause ? (t = now, 1) : 0; 94 | }); 95 | }; 96 | 97 | prototype.debounce = function(delay) { 98 | var s = stream(), evt = null, tid = null; 99 | 100 | function callback() { 101 | var df = evt.dataflow; 102 | s.receive(evt); 103 | evt = null; tid = null; 104 | if (df && df.run) df.run(); 105 | } 106 | 107 | this.targets().add(stream(null, null, function(e) { 108 | evt = e; 109 | if (tid) clearTimeout(tid); 110 | tid = setTimeout(callback, delay); 111 | })); 112 | 113 | return s; 114 | }; 115 | 116 | prototype.between = function(a, b) { 117 | var active = false; 118 | a.targets().add(stream(null, null, function() { active = true; })); 119 | b.targets().add(stream(null, null, function() { active = false; })); 120 | return this.filter(function() { return active; }); 121 | }; 122 | -------------------------------------------------------------------------------- /src/Dataflow.js: -------------------------------------------------------------------------------- 1 | import add from './dataflow/add'; 2 | import connect from './dataflow/connect'; 3 | import events from './dataflow/events'; 4 | import on from './dataflow/on'; 5 | import {rank, rerank} from './dataflow/rank'; 6 | import {pulse, touch, update} from './dataflow/update'; 7 | import {ingest, request, loadOptions} from './dataflow/load'; 8 | import {run, runAsync, runAfter, enqueue, getPulse} from './dataflow/run'; 9 | import changeset from './ChangeSet'; 10 | import Heap from './util/Heap'; 11 | import UniqueList from './util/UniqueList'; 12 | import {id, logger} from 'vega-util'; 13 | 14 | /** 15 | * A dataflow graph for reactive processing of data streams. 16 | * @constructor 17 | */ 18 | export default function Dataflow() { 19 | this._log = logger(); 20 | 21 | this._clock = 0; 22 | this._rank = 0; 23 | 24 | this._touched = UniqueList(id); 25 | this._pulses = {}; 26 | this._pulse = null; 27 | 28 | this._heap = new Heap(function(a, b) { return a.qrank - b.qrank; }); 29 | this._postrun = []; 30 | } 31 | 32 | var prototype = Dataflow.prototype; 33 | 34 | /** 35 | * The current timestamp of this dataflow. This value reflects the 36 | * timestamp of the previous dataflow run. The dataflow is initialized 37 | * with a stamp value of 0. The initial run of the dataflow will have 38 | * a timestap of 1, and so on. This value will match the 39 | * {@link Pulse.stamp} property. 40 | * @return {number} - The current timestamp value. 41 | */ 42 | prototype.stamp = function() { 43 | return this._clock; 44 | }; 45 | 46 | // OPERATOR REGISTRATION 47 | prototype.add = add; 48 | prototype.connect = connect; 49 | prototype.rank = rank; 50 | prototype.rerank = rerank; 51 | 52 | // OPERATOR UPDATES 53 | prototype.pulse = pulse; 54 | prototype.touch = touch; 55 | prototype.update = update; 56 | prototype.changeset = changeset; 57 | 58 | // DATA LOADING 59 | prototype.ingest = ingest; 60 | prototype.request = request; 61 | prototype.loadOptions = loadOptions; 62 | 63 | // EVENT HANDLING 64 | prototype.events = events; 65 | prototype.on = on; 66 | 67 | // PULSE PROPAGATION 68 | prototype.run = run; 69 | prototype.runAsync = runAsync; 70 | prototype.runAfter = runAfter; 71 | prototype._enqueue = enqueue; 72 | prototype._getPulse = getPulse; 73 | 74 | // LOGGING AND ERROR HANDLING 75 | 76 | function logMethod(method) { 77 | return function() { 78 | return this._log[method].apply(this, arguments); 79 | }; 80 | } 81 | 82 | /** 83 | * Logs a warning message. By default, logged messages are written to console 84 | * output. The message will only be logged if the current log level is high 85 | * enough to permit warning messages. 86 | */ 87 | prototype.warn = logMethod('warn'); 88 | 89 | /** 90 | * Logs a information message. By default, logged messages are written to 91 | * console output. The message will only be logged if the current log level is 92 | * high enough to permit information messages. 93 | */ 94 | prototype.info = logMethod('info'); 95 | 96 | /** 97 | * Logs a debug message. By default, logged messages are written to console 98 | * output. The message will only be logged if the current log level is high 99 | * enough to permit debug messages. 100 | */ 101 | prototype.debug = logMethod('debug'); 102 | 103 | /** 104 | * Get or set the current log level. If an argument is provided, it 105 | * will be used as the new log level. 106 | * @param {number} [level] - Should be one of None, Warn, Info 107 | * @return {number} - The current log level. 108 | */ 109 | prototype.logLevel = logMethod('level'); 110 | 111 | /** 112 | * Handle an error. By default, this method re-throws the input error. 113 | * This method can be overridden for custom error handling. 114 | */ 115 | prototype.error = function(err) { 116 | throw err; 117 | }; 118 | -------------------------------------------------------------------------------- /src/transforms/Impute.js: -------------------------------------------------------------------------------- 1 | import Transform from '../Transform'; 2 | import {ingest} from '../Tuple'; 3 | import {accessorName, error, inherits} from 'vega-util'; 4 | import {mean, min, max, median} from 'd3-array'; 5 | 6 | var Methods = { 7 | value: 'value', 8 | median: median, 9 | mean: mean, 10 | min: min, 11 | max: max 12 | }; 13 | 14 | var Empty = []; 15 | 16 | /** 17 | * Impute missing values. 18 | * @constructor 19 | * @param {object} params - The parameters for this operator. 20 | * @param {function(object): *} params.field - The value field to impute. 21 | * @param {Array} [params.groupby] - An array of 22 | * accessors to determine series within which to perform imputation. 23 | * @param {Array} [params.orderby] - An array of 24 | * accessors to determine the ordering within a series. 25 | * @param {string} [method='value'] - The imputation method to use. One of 26 | * 'value', 'mean', 'median', 'max', 'min'. 27 | * @param {*} [value=0] - The constant value to use for imputation 28 | * when using method 'value'. 29 | */ 30 | export default function Impute(params) { 31 | Transform.call(this, [], params); 32 | } 33 | 34 | var prototype = inherits(Impute, Transform); 35 | 36 | function getValue(_) { 37 | var m = _.method || Methods.value, v; 38 | 39 | if (Methods[m] == null) { 40 | error('Unrecognized imputation method: ' + m); 41 | } else if (m === Methods.value) { 42 | v = _.value !== undefined ? _.value : 0; 43 | return function() { return v; }; 44 | } else { 45 | return Methods[m]; 46 | } 47 | } 48 | 49 | function getField(_) { 50 | var f = _.field; 51 | return function(t) { return t ? f(t) : NaN; }; 52 | } 53 | 54 | prototype.transform = function(_, pulse) { 55 | var out = pulse.fork(pulse.ALL), 56 | impute = getValue(_), 57 | field = getField(_), 58 | fName = accessorName(_.field), 59 | gNames = _.groupby.map(accessorName), 60 | oNames = _.orderby.map(accessorName), 61 | groups = partition(pulse.source, _.groupby, _.orderby), 62 | curr = [], 63 | prev = this.value, 64 | m = groups.domain.length, 65 | group, value, gVals, oVals, g, i, j, l, n, t; 66 | 67 | for (g=0, l=groups.length; g= Info) { 33 | dt = Date.now(); 34 | df.debug('-- START PROPAGATION (' + df._clock + ') -----'); 35 | } 36 | 37 | // initialize queue, reset touched operators 38 | df._touched.forEach(function(op) { df._enqueue(op, true); }); 39 | df._touched = UniqueList(id); 40 | 41 | try { 42 | while (df._heap.size() > 0) { 43 | op = df._heap.pop(); 44 | 45 | // re-queue if rank changes 46 | if (op.rank !== op.qrank) { df._enqueue(op, true); continue; } 47 | 48 | // otherwise, evaluate the operator 49 | next = op.run(df._getPulse(op, encode)); 50 | 51 | if (level >= Debug) { 52 | df.debug(op.id, next === StopPropagation ? 'STOP' : next, op); 53 | } 54 | 55 | // propagate the pulse 56 | if (next !== StopPropagation) { 57 | df._pulse = next; 58 | if (op._targets) op._targets.forEach(function(op) { df._enqueue(op); }); 59 | } 60 | 61 | // increment visit counter 62 | ++count; 63 | } 64 | } catch (err) { 65 | df.error(err); 66 | } 67 | 68 | // reset pulse map 69 | df._pulses = {}; 70 | df._pulse = null; 71 | 72 | if (level >= Info) { 73 | dt = Date.now() - dt; 74 | df.info('> Pulse ' + df._clock + ': ' + count + ' operators; ' + dt + 'ms'); 75 | } 76 | 77 | // invoke callbacks queued via runAfter 78 | if (df._postrun.length) { 79 | var postrun = df._postrun; 80 | df._postrun = []; 81 | postrun.forEach(function(f) { 82 | try { f(df); } catch (err) { df.error(err); } 83 | }); 84 | } 85 | 86 | return count; 87 | } 88 | 89 | /** 90 | * Runs the dataflow and returns a Promise that resolves when the 91 | * propagation cycle completes. The standard run method may exit early 92 | * if there are pending data loading operations. In contrast, this 93 | * method returns a Promise to allow callers to receive notification 94 | * when dataflow evaluation completes. 95 | * @return {Promise} - A promise that resolves to this dataflow. 96 | */ 97 | export function runAsync() { 98 | return this._pending || Promise.resolve(this.run()); 99 | } 100 | 101 | /** 102 | * Schedules a callback function to be invoked after the current pulse 103 | * propagation completes. If no propagation is currently occurring, 104 | * the function is invoked immediately. 105 | * @param {function(Dataflow)} callback - The callback function to run. 106 | * The callback will be invoked with this Dataflow instance as its 107 | * sole argument. 108 | */ 109 | export function runAfter(callback) { 110 | if (this._pulse) { 111 | // pulse propagation is currently running, queue to run after 112 | this._postrun.push(callback); 113 | } else { 114 | // pulse propagation already complete, invoke immediately 115 | try { callback(this); } catch (err) { this.error(err); } 116 | } 117 | } 118 | 119 | /** 120 | * Enqueue an operator into the priority queue for evaluation. The operator 121 | * will be enqueued if it has no registered pulse for the current cycle, or if 122 | * the force argument is true. Upon enqueue, this method also sets the 123 | * operator's qrank to the current rank value. 124 | */ 125 | export function enqueue(op, force) { 126 | var p = !this._pulses[op.id]; 127 | if (p) this._pulses[op.id] = this._pulse; 128 | if (p || force) { 129 | op.qrank = op.rank; 130 | this._heap.push(op); 131 | } 132 | } 133 | 134 | /** 135 | * Provide a correct pulse for evaluating an operator. If the operator has an 136 | * explicit source operator, we will try to pull the pulse(s) from it. 137 | * If there is an array of source operators, we build a multi-pulse. 138 | * Otherwise, we return a current pulse with correct source data. 139 | * If the pulse is the pulse map has an explicit target set, we use that. 140 | * Else if the pulse on the upstream source operator is current, we use that. 141 | * Else we use the pulse from the pulse map, but copy the source tuple array. 142 | */ 143 | export function getPulse(op, encode) { 144 | var s = op.source, 145 | stamp = this._clock, 146 | p; 147 | 148 | if (s && isArray(s)) { 149 | p = s.map(function(_) { return _.pulse; }); 150 | return new MultiPulse(this, stamp, p, encode); 151 | } else { 152 | s = s && s.pulse; 153 | p = this._pulses[op.id]; 154 | if (s && s !== StopPropagation) { 155 | if (s.stamp === stamp && p.target !== op) p = s; 156 | else p.source = s.source; 157 | } 158 | return p; 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /src/transforms/aggregate/Measures.js: -------------------------------------------------------------------------------- 1 | import {extend, identity} from 'vega-util'; 2 | 3 | export var Aggregates = { 4 | 'values': measure({ 5 | name: 'values', 6 | init: 'cell.store = true;', 7 | set: 'cell.data.values()', idx: -1 8 | }), 9 | 'count': measure({ 10 | name: 'count', 11 | set: 'cell.num' 12 | }), 13 | 'missing': measure({ 14 | name: 'missing', 15 | set: 'this.missing' 16 | }), 17 | 'valid': measure({ 18 | name: 'valid', 19 | set: 'this.valid' 20 | }), 21 | 'distinct': measure({ 22 | name: 'distinct', 23 | init: 'this.dmap = {}; this.distinct = 0;', 24 | add: 'this.dmap[v] = 1 + (this.dmap[v] || (++this.distinct, 0));', 25 | rem: 'if (!(--this.dmap[v])) --this.distinct;', 26 | set: 'this.distinct' 27 | }), 28 | 'sum': measure({ 29 | name: 'sum', 30 | init: 'this.sum = 0;', 31 | add: 'this.sum += v;', 32 | rem: 'this.sum -= v;', 33 | set: 'this.sum' 34 | }), 35 | 'mean': measure({ 36 | name: 'mean', 37 | init: 'this.mean = 0;', 38 | add: 'var d = v - this.mean; this.mean += d / this.valid;', 39 | rem: 'var d = v - this.mean; this.mean -= this.valid ? d / this.valid : this.mean;', 40 | set: 'this.mean' 41 | }), 42 | 'average': measure({ 43 | name: 'average', 44 | set: 'this.mean', 45 | req: ['mean'], idx: 1 46 | }), 47 | 'variance': measure({ 48 | name: 'variance', 49 | init: 'this.dev = 0;', 50 | add: 'this.dev += d * (v - this.mean);', 51 | rem: 'this.dev -= d * (v - this.mean);', 52 | set: 'this.valid > 1 ? this.dev / (this.valid-1) : 0', 53 | req: ['mean'], idx: 1 54 | }), 55 | 'variancep': measure({ 56 | name: 'variancep', 57 | set: 'this.valid > 1 ? this.dev / this.valid : 0', 58 | req: ['variance'], idx: 2 59 | }), 60 | 'stdev': measure({ 61 | name: 'stdev', 62 | set: 'this.valid > 1 ? Math.sqrt(this.dev / (this.valid-1)) : 0', 63 | req: ['variance'], idx: 2 64 | }), 65 | 'stdevp': measure({ 66 | name: 'stdevp', 67 | set: 'this.valid > 1 ? Math.sqrt(this.dev / this.valid) : 0', 68 | req: ['variance'], idx: 2 69 | }), 70 | 'stderr': measure({ 71 | name: 'stderr', 72 | set: 'this.valid > 1 ? Math.sqrt(this.dev / (this.valid * (this.valid-1))) : 0', 73 | req: ['variance'], idx: 2 74 | }), 75 | 'ci0': measure({ 76 | name: 'ci0', 77 | set: 'cell.data.ci0(this.get)', 78 | req: ['values'], idx: 3 79 | }), 80 | 'ci1': measure({ 81 | name: 'ci1', 82 | set: 'cell.data.ci1(this.get)', 83 | req: ['values'], idx: 3 84 | }), 85 | 'median': measure({ 86 | name: 'median', 87 | set: 'cell.data.q2(this.get)', 88 | req: ['values'], idx: 3 89 | }), 90 | 'q1': measure({ 91 | name: 'q1', 92 | set: 'cell.data.q1(this.get)', 93 | req: ['values'], idx: 3 94 | }), 95 | 'q3': measure({ 96 | name: 'q3', 97 | set: 'cell.data.q3(this.get)', 98 | req: ['values'], idx: 3 99 | }), 100 | 'argmin': measure({ 101 | name: 'argmin', 102 | add: 'if (v < this.min) this.argmin = t;', 103 | rem: 'if (v <= this.min) this.argmin = null;', 104 | set: 'this.argmin || cell.data.argmin(this.get)', 105 | req: ['min'], str: ['values'], idx: 3 106 | }), 107 | 'argmax': measure({ 108 | name: 'argmax', 109 | add: 'if (v > this.max) this.argmax = t;', 110 | rem: 'if (v >= this.max) this.argmax = null;', 111 | set: 'this.argmax || cell.data.argmax(this.get)', 112 | req: ['max'], str: ['values'], idx: 3 113 | }), 114 | 'min': measure({ 115 | name: 'min', 116 | init: 'this.min = null;', 117 | add: 'if (v < this.min || this.min === null) this.min = v;', 118 | rem: 'if (v <= this.min) this.min = NaN;', 119 | set: 'this.min = (isNaN(this.min) ? cell.data.min(this.get) : this.min)', 120 | str: ['values'], idx: 4 121 | }), 122 | 'max': measure({ 123 | name: 'max', 124 | init: 'this.max = null;', 125 | add: 'if (v > this.max || this.max === null) this.max = v;', 126 | rem: 'if (v >= this.max) this.max = NaN;', 127 | set: 'this.max = (isNaN(this.max) ? cell.data.max(this.get) : this.max)', 128 | str: ['values'], idx: 4 129 | }) 130 | }; 131 | 132 | export function createMeasure(op, name) { 133 | return Aggregates[op](name); 134 | } 135 | 136 | function measure(base) { 137 | return function(out) { 138 | var m = extend({init:'', add:'', rem:'', idx:0}, base); 139 | m.out = out || base.name; 140 | return m; 141 | }; 142 | } 143 | 144 | function compareIndex(a, b) { 145 | return a.idx - b.idx; 146 | } 147 | 148 | function resolve(agg, stream) { 149 | function collect(m, a) { 150 | function helper(r) { if (!m[r]) collect(m, m[r] = Aggregates[r]()); } 151 | if (a.req) a.req.forEach(helper); 152 | if (stream && a.str) a.str.forEach(helper); 153 | return m; 154 | } 155 | var map = agg.reduce( 156 | collect, 157 | agg.reduce(function(m, a) { return (m[a.name] = a, m); }, {}) 158 | ); 159 | var values = [], key; 160 | for (key in map) values.push(map[key]); 161 | return values.sort(compareIndex); 162 | } 163 | 164 | export function compileMeasures(agg, field) { 165 | var get = field || identity, 166 | all = resolve(agg, true), // assume streaming removes may occur 167 | ctr = 'this.cell = cell; this.tuple = t; this.valid = 0; this.missing = 0;', 168 | add = 'if(v==null){this.missing++; return;} if(v!==v) return; ++this.valid;', 169 | rem = 'if(v==null){this.missing--; return;} if(v!==v) return; --this.valid;', 170 | set = 'var t = this.tuple; var cell = this.cell;'; 171 | 172 | all.forEach(function(a) { 173 | if (a.idx < 0) { 174 | ctr = a.init + ctr; 175 | add = a.add + add; 176 | rem = a.rem + rem; 177 | } else { 178 | ctr += a.init; 179 | add += a.add; 180 | rem += a.rem; 181 | } 182 | }); 183 | agg.slice().sort(compareIndex).forEach(function(a) { 184 | set += 't[\'' + a.out + '\']=' + a.set + ';'; 185 | }); 186 | set += 'return t;'; 187 | 188 | ctr = Function('cell', 't', ctr); 189 | ctr.prototype.add = Function('v', 't', add); 190 | ctr.prototype.rem = Function('v', 't', rem); 191 | ctr.prototype.set = Function(set); 192 | ctr.prototype.get = get; 193 | ctr.fields = agg.map(function(_) { return _.out; }); 194 | return ctr; 195 | } 196 | -------------------------------------------------------------------------------- /src/Operator.js: -------------------------------------------------------------------------------- 1 | import Parameters from './Parameters'; 2 | import UniqueList from './util/UniqueList'; 3 | import {array, error, id, isArray} from 'vega-util'; 4 | 5 | var OP_ID = 0; 6 | var PULSE = 'pulse'; 7 | var NO_PARAMS = new Parameters(); 8 | 9 | // Boolean Flags 10 | var SKIP = 1, 11 | MODIFIED = 2; 12 | 13 | /** 14 | * An Operator is a processing node in a dataflow graph. 15 | * Each operator stores a value and an optional value update function. 16 | * Operators can accept a hash of named parameters. Parameter values can 17 | * either be direct (JavaScript literals, arrays, objects) or indirect 18 | * (other operators whose values will be pulled dynamically). Operators 19 | * included as parameters will have this operator added as a dependency. 20 | * @constructor 21 | * @param {*} [init] - The initial value for this operator. 22 | * @param {function(object, Pulse)} [update] - An update function. Upon 23 | * evaluation of this operator, the update function will be invoked and the 24 | * return value will be used as the new value of this operator. 25 | * @param {object} [params] - The parameters for this operator. 26 | * @param {boolean} [react=true] - Flag indicating if this operator should 27 | * listen for changes to upstream operators included as parameters. 28 | * @see parameters 29 | */ 30 | export default function Operator(init, update, params, react) { 31 | this.id = ++OP_ID; 32 | this.value = init; 33 | this.stamp = -1; 34 | this.rank = -1; 35 | this.qrank = -1; 36 | this.flags = 0; 37 | 38 | if (update) { 39 | this._update = update; 40 | } 41 | if (params) this.parameters(params, react); 42 | } 43 | 44 | var prototype = Operator.prototype; 45 | 46 | /** 47 | * Returns a list of target operators dependent on this operator. 48 | * If this list does not exist, it is created and then returned. 49 | * @return {UniqueList} 50 | */ 51 | prototype.targets = function() { 52 | return this._targets || (this._targets = UniqueList(id)); 53 | }; 54 | 55 | /** 56 | * Sets the value of this operator. 57 | * @param {*} value - the value to set. 58 | * @return {Number} Returns 1 if the operator value has changed 59 | * according to strict equality, returns 0 otherwise. 60 | */ 61 | prototype.set = function(value) { 62 | return this.value !== value ? (this.value = value, 1) : 0; 63 | }; 64 | 65 | function flag(bit) { 66 | return function(state) { 67 | var f = this.flags; 68 | if (arguments.length === 0) return !!(f & bit); 69 | this.flags = state ? (f | bit) : (f & ~bit); 70 | return this; 71 | }; 72 | } 73 | 74 | /** 75 | * Indicates that operator evaluation should be skipped on the next pulse. 76 | * This operator will still propagate incoming pulses, but its update function 77 | * will not be invoked. The skip flag is reset after every pulse, so calling 78 | * this method will affect processing of the next pulse only. 79 | */ 80 | prototype.skip = flag(SKIP); 81 | 82 | /** 83 | * Indicates that this operator's value has been modified on its most recent 84 | * pulse. Normally modification is checked via strict equality; however, in 85 | * some cases it is more efficient to update the internal state of an object. 86 | * In those cases, the modified flag can be used to trigger propagation. Once 87 | * set, the modification flag persists across pulses until unset. The flag can 88 | * be used with the last timestamp to test if a modification is recent. 89 | */ 90 | prototype.modified = flag(MODIFIED); 91 | 92 | /** 93 | * Sets the parameters for this operator. The parameter values are analyzed for 94 | * operator instances. If found, this operator will be added as a dependency 95 | * of the parameterizing operator. Operator values are dynamically marshalled 96 | * from each operator parameter prior to evaluation. If a parameter value is 97 | * an array, the array will also be searched for Operator instances. However, 98 | * the search does not recurse into sub-arrays or object properties. 99 | * @param {object} params - A hash of operator parameters. 100 | * @param {boolean} [react=true] - A flag indicating if this operator should 101 | * automatically update (react) when parameter values change. In other words, 102 | * this flag determines if the operator registers itself as a listener on 103 | * any upstream operators included in the parameters. 104 | * @return {Operator[]} - An array of upstream dependencies. 105 | */ 106 | prototype.parameters = function(params, react) { 107 | react = react !== false; 108 | var self = this, 109 | argval = (self._argval = self._argval || new Parameters()), 110 | argops = (self._argops = self._argops || []), 111 | deps = [], 112 | name, value, n, i; 113 | 114 | function add(name, index, value) { 115 | if (value instanceof Operator) { 116 | if (value !== self) { 117 | if (react) value.targets().add(self); 118 | deps.push(value); 119 | } 120 | argops.push({op:value, name:name, index:index}); 121 | } else { 122 | argval.set(name, index, value); 123 | } 124 | } 125 | 126 | for (name in params) { 127 | value = params[name]; 128 | 129 | if (name === PULSE) { 130 | array(value).forEach(function(op) { 131 | if (!(op instanceof Operator)) { 132 | error('Pulse parameters must be operator instances.'); 133 | } else if (op !== self) { 134 | op.targets().add(self); 135 | deps.push(op); 136 | } 137 | }); 138 | self.source = value; 139 | } else if (isArray(value)) { 140 | argval.set(name, -1, Array(n = value.length)); 141 | for (i=0; i} params.groupby - An array of accessors to groupby. 19 | * @param {Array} params.fields - An array of accessors to aggregate. 20 | * @param {Array} params.ops - An array of strings indicating aggregation operations. 21 | * @param {Array} [params.as] - An array of output field names for aggregated values. 22 | * @param {boolean} [params.drop=true] - A flag indicating if empty cells should be removed. 23 | */ 24 | export default function Aggregate(params) { 25 | Transform.call(this, null, params); 26 | 27 | this._adds = []; // array of added output tuples 28 | this._mods = []; // array of modified output tuples 29 | this._alen = 0; // number of active added tuples 30 | this._mlen = 0; // number of active modified tuples 31 | this._drop = true; // should empty aggregation cells be removed 32 | 33 | this._dims = []; // group-by dimension accessors 34 | this._dnames = []; // group-by dimension names 35 | 36 | this._measures = []; // collection of aggregation monoids 37 | this._countOnly = false; // flag indicating only count aggregation 38 | this._counts = null; // collection of count fields 39 | this._prev = null; // previous aggregation cells 40 | 41 | this._inputs = null; // array of dependent input tuple field names 42 | this._outputs = null; // array of output tuple field names 43 | } 44 | 45 | var prototype = inherits(Aggregate, Transform); 46 | 47 | prototype.transform = function(_, pulse) { 48 | var aggr = this, 49 | out = pulse.fork(pulse.NO_SOURCE | pulse.NO_FIELDS), 50 | mod; 51 | 52 | this.stamp = out.stamp; 53 | 54 | if (this.value && ((mod = _.modified()) || pulse.modified(this._inputs))) { 55 | this._prev = this.value; 56 | this.value = mod ? this.init(_) : {}; 57 | pulse.visit(pulse.SOURCE, function(t) { aggr.add(t); }); 58 | } else { 59 | this.value = this.value || this.init(_); 60 | pulse.visit(pulse.REM, function(t) { aggr.rem(t); }); 61 | pulse.visit(pulse.ADD, function(t) { aggr.add(t); }); 62 | } 63 | 64 | // Indicate output fields and return aggregate tuples. 65 | out.modifies(this._outputs); 66 | 67 | aggr._drop = _.drop !== false; 68 | return aggr.changes(out); 69 | }; 70 | 71 | prototype.init = function(_) { 72 | // initialize input and output fields 73 | var inputs = (this._inputs = []), 74 | outputs = (this._outputs = []), 75 | inputMap = {}; 76 | 77 | function inputVisit(get) { 78 | var fields = get.fields, i = 0, n = fields.length, f; 79 | for (; i