├── .codeclimate.yml ├── .eslintrc.js ├── .gitignore ├── .travis.yml ├── CONTRIBUTORS ├── LICENSE ├── NOTES.md ├── README.md ├── package.json ├── src ├── aggregation.js ├── clear.js ├── column.js ├── crossfilter.js ├── destroy.js ├── dimension.js ├── expressions.js ├── filters.js ├── lodash.js ├── postAggregation.js ├── query.js ├── reductioAggregators.js ├── reductiofy.js └── universe.js ├── test ├── clear.spec.js ├── column.spec.js ├── destroy.spec.js ├── filter-all.spec.js ├── filter.spec.js ├── fixtures │ └── data.js ├── post-aggregation.spec.js ├── query.dynamicData.spec.js ├── query.spec.js └── universe.spec.js └── yarn.lock /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | eslint: 3 | enabled: true 4 | checks: 5 | comma-dangle: 6 | enabled: false 7 | ratings: 8 | paths: 9 | - src/** 10 | exclude_paths: 11 | - test/** 12 | - universe.js 13 | - universe.min.js 14 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true, 4 | commonjs: true, 5 | es6: true 6 | }, 7 | // extends: "eslint:recommended", 8 | parserOptions: { 9 | sourceType: 'module' 10 | }, 11 | rules: { 12 | indent: ['error', 2], 13 | 'linebreak-style': ['error', 'unix'], 14 | quotes: ['error', 'single'], 15 | semi: ['error', 'never'], 16 | 'comma-dangle': ['error', 'always-multiline'] 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | npm-debug.log 3 | .coverrun 4 | coverage/coverage.html 5 | /universe.js 6 | /universe.min.js 7 | 8 | npm-debug.log* 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "4.2" 4 | script: npm test 5 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The following people have contributed to universe 2 | 3 | Tanner Linsley - https://github.com/crossfilter/universe/commits?author=tannerlinsley 4 | Jayson Harshbarger - https://github.com/crossfilter/universe/commits?author=hypercubed 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 crossfilter 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /NOTES.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/crossfilter/universe/5369fc2e1c657afeb1dee8d0156829e7e959d2e8/NOTES.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Universe 2 | [![Join the chat at https://gitter.im/crossfilter/universe](https://badges.gitter.im/crossfilter/universe.svg)](https://gitter.im/crossfilter/universe?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 3 | 4 | [![Build Status](https://travis-ci.org/crossfilter/universe.svg?branch=master)](https://travis-ci.org/crossfilter/universe) [![Code Climate](https://codeclimate.com/github/crossfilter/universe/badges/gpa.svg)](https://codeclimate.com/github/crossfilter/universe) 5 | 6 | ## The easiest and fastest way to explore your data 7 | Before Universe, exploring and filtering large datasets in javascript meant constant data looping, complicated indexing, and countless lines of code to dissect your data. 8 | 9 | With Universe, you can be there in just a few lines of code. You've got better things to do than write intense map-reduce functions or learn the intricate inner-workings of [Crossfilter](https://github.com/crossfilter/crossfilter) ;) 10 | 11 | ## Features 12 | - Simple, yet powerful query syntax 13 | - Built on, and tightly integrated with [Crossfilter](https://github.com/crossfilter/crossfilter), and [Reductio](https://github.com/crossfilter/reductio) - the fastest multi-dimensional JS data frameworks available 14 | - Real-time updates to query results as you filter 15 | - Flexible filtering system 16 | - Automatic and invisible management of data indexing and memory 17 | - Post Aggregation 18 | 19 | ## Features in the Pipeline 20 | - Query Joins 21 | - Query Macros 22 | - Sub Queries 23 | - To help contribute, join us at [![Join the chat at https://gitter.im/crossfilter/universe](https://badges.gitter.im/crossfilter/universe.svg)](https://gitter.im/crossfilter/universe?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 24 | 25 | ## Demos 26 | - [Basic Usage](http://codepen.io/tannerlinsley/pen/oxjyvg?editors=0010) (Codepen) 27 | 28 | ## [API](#api) 29 | 30 | - [universe()](#api-universe) 31 | - [.query()](#api-query) 32 | - [.filter()](#api-filter) 33 | - [.filterAll()](#api-filterAll) 34 | - [.column()](#api-column) 35 | - [.clear()](#api-clear) 36 | - [.add()](#api-add) 37 | - [.remove()](#api-remove) 38 | 39 | 40 | - [Post Aggregation](#post-aggregation) 41 | - [.post()](#post-aggregation-post) 42 | - [Pro Tips](#pro-tips) 43 | 44 | ## Getting Started 45 | ### Installation 46 | **NPM** 47 | 48 | ```shell 49 | npm install universe --save 50 | ``` 51 | 52 | **CDN or Download** from the [npmcdn](https://npmcdn.com/) load or download [universe.js](https://npmcdn.com/universe@latest/universe.js) or [universe.min.js](https://npmcdn.com/universe@latest/universe.min.js) file as part of your application. 53 | 54 | ### Create a new Universe 55 | Pass `universe` an array of objects or a Crossfilter instance: 56 | 57 | ```javascript 58 | 59 | var universe = require('universe') 60 | 61 | var myUniverse = universe([ 62 | {date: "2011-11-14T16:17:54Z", quantity: 2, total: 190, tip: 100, type: "tab", productIDs: ["001"]}, 63 | {date: "2011-11-14T16:20:19Z", quantity: 2, total: 190, tip: 100, type: "tab", productIDs: ["001", "005"]}, 64 | {date: "2011-11-14T16:28:54Z", quantity: 1, total: 300, tip: 200, type: "visa", productIDs: ["004", "005"]}, 65 | ... 66 | ]) 67 | .then(function(myUniverse){ 68 | // And now you're ready to query! :) 69 | return myUniverse 70 | }) 71 | ``` 72 | 73 | ### Query your data 74 | ```javascript 75 | 76 | .then(function(myUniverse){ 77 | myUniverse.query({ 78 | groupBy: 'type' // GroupBy the type key 79 | columns: { 80 | $count: true, // Count the number of records 81 | quantity: { // Create a custom 'quantity' column 82 | $sum: 'quantity' // Sum the quantity column 83 | }, 84 | }, 85 | // Limit selection to rows where quantity is greater than 50 86 | filter: { 87 | quantity: { 88 | $gt: 50 89 | } 90 | }, 91 | }) 92 | 93 | // Optionally post-aggregate your data 94 | // Reduce all results after 5 to a single result using sums 95 | myUniverse.squash(5, null, { 96 | count: '$sum', 97 | quantity: { 98 | sum: '$sum' 99 | } 100 | }) 101 | 102 | // See Post-Aggregations for more information 103 | }) 104 | ``` 105 | 106 | ### Use your data 107 | 108 | ```javascript 109 | .then(function(res) { 110 | // Use your data for tables, charts, data visualiztion, etc. 111 | res.data === [ 112 | {"key": "cash","value": {"count": 2,"quantity": {"sum": 3}}}, 113 | {"key": "tab","value": {"count": 8,"quantity": {"sum": 16}}}, 114 | {"key": "visa","value": {"count": 2,"quantity": {"sum": 2}}} 115 | ] 116 | 117 | // Or plost the data in DC.js using the underlying crossfilter dimension and group 118 | dc.pieChart('#chart') 119 | .dimension(res.dimension) 120 | .group(res.group) 121 | 122 | // Pass the query's universe instance to keep chaining 123 | return res.universe 124 | }) 125 | ``` 126 | 127 | ### Explore your data 128 | 129 | As you filter your data on the universe level, every query's result is updated in real-time to reflect changes in aggregation 130 | 131 | ```javascript 132 | // Filter records where 'type' === 'visa' 133 | .then(function(myUniverse) { 134 | return myUniverse.filter('type', 'visa') 135 | }) 136 | 137 | // Filter records where 'type' === 'visa' or 'tab' 138 | .then(function(myUniverse) { 139 | return myUniverse.filter('type', ['visa', 'tab']) 140 | }) 141 | 142 | // Filter records where 'total' is between 50 and 100 143 | .then(function(myUniverse) { 144 | return myUniverse.filter('total', [50, 10], true) 145 | }) 146 | 147 | // Filter records using an expressive and JSON friendly query syntax 148 | .then(function(myUniverse) { 149 | return myUniverse.filter('total', { 150 | $lt: { // Filter to results where total is less than 151 | '$get(total)': { // the "total" property from 152 | '$nthLast(3)': { // the 3rd to the last row from 153 | $column: 'date' // the dataset sorted by the date column 154 | } 155 | } 156 | } 157 | }) 158 | }) 159 | 160 | // Or if you're feeling powerful, just write your own custom filter function 161 | .then(function(myUniverse){ 162 | return myUniverse.filter({ 163 | total: function(row){ 164 | return (row.quantity * row.sum) > 50 165 | } 166 | }) 167 | }) 168 | 169 | // Clear the filters for the 'type' column 170 | .then(function(myUniverse){ 171 | return myUniverse.filter('type') 172 | }) 173 | 174 | // Apply many filters in one go 175 | .then(function(myUniverse){ 176 | return myUniverse.filterAll([{ 177 | column: 'type', 178 | value: 'visa', 179 | }, { 180 | column: 'quantity', 181 | value: [200, 500], 182 | isRange: true, 183 | }]) 184 | }) 185 | 186 | // Clear all of the filters 187 | .then(function(myUniverse){ 188 | return myUniverse.filterAll() 189 | }) 190 | ``` 191 | 192 | ### Clean Up 193 | 194 | ```javascript 195 | 196 | // Remove a column index 197 | .then(function(myUniverse){ 198 | return myUniverse.clear('total') 199 | }) 200 | 201 | // Remove all columns 202 | .then(function(myUniverse){ 203 | return myUniverse.clear() 204 | }) 205 | ``` 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 |

API #

220 | 221 |

universe( [data] , {config} ) #

222 | 223 | - Description 224 | - Creates a new universe instance 225 | - Parameters 226 | - `[data]` - An array of objects 227 | - `{config}` - Optional configurations for this Universe instance 228 | - `{generatedColumns}` - An object of keys and their respective accessor functions used to dynamically generate columns. 229 | - Returns a `promise` that is resolved with the **universe instance** 230 | 231 | - [Example](Create a new Universe) 232 | - Generated Columns Example 233 | ```javascript 234 | universe([ 235 | {day: '1', price: 30, quantity: 3}, 236 | {day: '2', price: 40, quantity: 5} 237 | ], { 238 | generatedColumns: { 239 | total: function(row){return row.price * row.quantity} 240 | } 241 | }) 242 | .then(function(myUniverse){ 243 | // data[0].total === 90 244 | // data[1].total === 200 245 | }) 246 | ``` 247 | 248 |

.query( {queryObject} ) #

249 | 250 | - Description 251 | - Creates a new query from a universe instance 252 | - Parameters 253 | - `queryObject`: 254 | - `groupBy` - Property name, property string representation, or even a function! (see `.column()` method), 255 | - `select` - An object of column aggregations and/or column names 256 | - `$aggregation` - Aggregations are prefixed with a `$` 257 | - `columnName` - Creates a nested column with the name provided 258 | - `filter` - A filter object that is applied to the query (similar to a `where` clause in mySQL) 259 | - Returns 260 | - `promise`, resolved with a **query results object** 261 | - `data` - The result of the query 262 | - `group` - The crossfilter/reductio group used to build the query 263 | - `dimension` - The crossfilter dimension used to build the query 264 | - `crossfilter` - The crossfilter that runs this universe 265 | - `universe` - The current instance of the universe. Return this to keep chaining via promises 266 | 267 | - [Example](#Explore your data) 268 | 269 |

.filter( columnKey, filterObject, isArray, replace ) #

270 | 271 | - Description 272 | - Filters everything in the universe to only include rows that match certain conditions. Queries automatically and instantaneously update their values and aggregations. 273 | - Parameters 274 | - `columnKey` - The object property to filter on, 275 | - Returns 276 | - `promise` resolved with 277 | - **universe instance** 278 | 279 | - [Example](#Query your data) 280 | 281 |

.filterAll() #

282 | 283 | - Description 284 | - Clears all filters accross all dimensiona. 285 | - Returns 286 | - `promise` resolved with 287 | - **universe instance** 288 | 289 |

.column( columnKey/columnObject ) #

290 | 291 | - Description 292 | - Use to optionally pre-index a column. Accepts either: 293 | - String or number corresponding to the key or index of the column. eg. `propertyName` or `2` 294 | - A nested string representation of the property. eg. `a.nested.property`, `a.nested[number]` 295 | - Multiple singular key shorthand eg. `['prop1', 'prop2', 'prop3']` 296 | - A callback function that returns the key (very powerful) eg. `function(d){return d.myProperty}` 297 | - Parameters 298 | - `columnKey` - the column property or array index you would like to pre-compile eg. 299 | ```javascript 300 | .then(function(universe){ 301 | return universe.column('total') 302 | }) 303 | ``` 304 | - `columnObject` allows you to override the column type, otherwise it is calculated automatically: 305 | ```javascript 306 | .then(function(universe){ 307 | return universe.column({ 308 | key: columnKey, 309 | type: 'number' 310 | }) 311 | }) 312 | ``` 313 | - Returns 314 | - `promise` resolved with 315 | - **universe instance** 316 | 317 | - [Example](#Pre-compile Columns) 318 | 319 |

.clear( columnKey/columnObject/[columnKeys/columnObjects] ) #

320 | 321 | - Description 322 | - Clears individual or all column definitions and indexes 323 | - Parameters 324 | - `columnKey` - the column property or array of columns you would like to clear eg. 325 | ```javascript 326 | .then(function(universe){ 327 | // Single Key 328 | return universe.clear('total') 329 | // Complex Key 330 | return universe.clear({key: ['complex', 'key']}) 331 | // Multiple Single Keys 332 | return universe.clear(['total', 'quantity']) 333 | // Multiple Complex Keys 334 | return universe.clear([{key: ['complex', 'key']}, {key: ['another', 'one']}]) 335 | }) 336 | ``` 337 | - Returns 338 | - `promise` resolved with 339 | - **universe instance** 340 | 341 | - [Example](#Clean Up) 342 | 343 |

.add( [data] ) #

344 | 345 | - Description 346 | - Adds additional data to a universe instance. This data will be indexed, aggregated and queries/filters immediately updated when added. 347 | - Parameters 348 | - `[data]` - An new array of objects similar to the original dataset 349 | - Returns 350 | - `promise` resolved with 351 | - **universe instance** 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 |

Post Aggregation #

366 | 367 | Post aggregation methods can be run on query results to further modify your data. Just like queries, the results magically and instantly respond to filtering. 368 | - Each post aggregation is very powerful, but not all post aggregations can be chained together. 369 | 370 | ### Locking a query 371 | A majority of the time, you're probably only interested in the end result of a query chain. For this reason, Post Aggregations default to mutating the data of their direct parent (unless the parent is the original query), thereby avoiding unnecessary copying of data. 372 | On the other hand, if you plan on accessing data at any point in the middle of a query chain, you will need to `lock()` that query's results. This ensure's it won't be overwritten or mutated by any further post aggregation. 373 | 374 | *Note:* Running more than 1 post aggregation on a query will automatically lock the parent query. 375 | 376 | ```javascript 377 | 378 | .then(function(universe){ 379 | return universe.query({ 380 | groupBy: 'tag' 381 | }) 382 | }) 383 | .then(function(query){ 384 | query.lock() 385 | var all = query.data 386 | return query.limit(5) 387 | }) 388 | .then(function(query){ 389 | var only5 = query.data 390 | 391 | all.length === 10 392 | only5.length === 5 393 | }) 394 | ``` 395 | Without locking the above query before using `.limit(5)`, the `all` data array would have been mutated by `.limit(5)` 396 | 397 |

.sortByKey(descending) #

398 | 399 | - Description 400 | - Sort results by key (ascending or descending) 401 | - Parameters 402 | - `descending` - Pass true to sortKeys in descending order 403 | ```javascript 404 | .then(function(query){ 405 | return query.sortByKey(true) 406 | }) 407 | ``` 408 | - Returns 409 | - `promise` resolved with 410 | - **query instance** 411 | 412 | 413 |

.limit(n, n2) #

414 | 415 | - Description 416 | - Limit results to those between`n` and `n2`. If `n2` is not passed, will limit to the first `n` records 417 | - Parameters 418 | - `n` - Start index. Defaults to 0 if `null` or `undefined`, 419 | - `n2` - End index. Defaults to `query.data.length` if `null`. If `undefined`, will limit to the first `n` records instead. 420 | ```javascript 421 | .then(function(query){ 422 | // limits results to the first 5 records 423 | return query.limit(5) 424 | // limits results to records 5 through 10 425 | return query.limit(4, 10) 426 | }) 427 | ``` 428 | - Returns 429 | - `promise` resolved with 430 | - **query instance** 431 | 432 | 433 |

.squash(n, n2, aggregationMap, keyName) #

434 | 435 | - Description 436 | - Takes records from `n` to `n2` and reduces them to a single record using the aggregationMap 437 | - Parameters 438 | - `n` - Start index. Defaults to `0` if `false`-y 439 | - `n2` - End index. Defaults to `query.data.length` if `false`-y 440 | - `aggregationMap` - A 1:1 map of property to the aggregation to be used when combining the records 441 | - `keyName` (optional) - The key to be used for the new record. Defaults to `Other` 442 | 443 | ```javascript 444 | .then(function(universe){ 445 | universe.query({ 446 | groupBy: 'type', 447 | select: { 448 | $sum: 'total', 449 | otherColumn: { 450 | $avg: 'tip' 451 | } 452 | }) 453 | }) 454 | .then(function(query){ 455 | // Will squash all records after the 5 record 456 | query.squash(5, null, { 457 | // Sum the sum column 458 | sum: '$sum', 459 | othercolumn: { 460 | // Average the avg column 461 | avg: '$avg' 462 | } 463 | }, 'Everything after 5') 464 | // Give the squashed record a new key 465 | }) 466 | ``` 467 | - Returns 468 | - `promise` resolved with 469 | - **query instance** 470 | 471 | 472 |

.change(n, n2, changeFields) #

473 | 474 | - Description 475 | - Determines the change from the `n` to `n2` using the keys in `changeFields` 476 | - Parameters 477 | - `n` - Start index. Defaults to `0` if `false`-y 478 | - `n2` - End index. Defaults to `query.data.length` if `false`-y 479 | - `changeFields` - An object or array, referencing the fields to measure for change 480 | 481 | ```javascript 482 | .then(function(universe){ 483 | universe.query({ 484 | groupBy: 'type', 485 | select: { 486 | $sum: 'total', 487 | otherColumn: { 488 | $avg: 'tip' 489 | } 490 | } 491 | }) 492 | }) 493 | .then(function(query){ 494 | // Measure the change for sum and avg from result 0 to 10 495 | query.change(0, 10, { 496 | sum: true 497 | otherColumn: { 498 | avg: true 499 | } 500 | }) 501 | }) 502 | ``` 503 | - Returns 504 | - `promise` resolved with 505 | - **query instance** 506 | - `query.data` is now an object: 507 | ```javascript 508 | { 509 | key: ['nKey', 'n2Key'], 510 | value: { 511 | sumChange: 7, 512 | otherColumn: { 513 | avgChange: 4 514 | } 515 | } 516 | } 517 | ``` 518 | 519 | 520 |

.changeMap(changeMapObj) #

521 | 522 | - Description 523 | - Determines incremental change for each record across the fields defined in `changeMapObj` 524 | - Parameters 525 | - `changeMapObj` - An object or array, referencing the fields to measure for change 526 | 527 | ```javascript 528 | .then(function(universe){ 529 | universe.query({ 530 | groupBy: 'type', 531 | select: { 532 | $sum: 'total', 533 | otherColumn: { 534 | $avg: 'tip' 535 | } 536 | } 537 | }) 538 | }) 539 | .then(function(query){ 540 | // Measure the change for sum and avg from result 0 to 10 541 | query.change({ 542 | sum: true 543 | otherColumn: { 544 | avg: true 545 | } 546 | }) 547 | }) 548 | ``` 549 | - Returns 550 | - `promise` resolved with 551 | - **query instance** 552 | - `query.data` records are now decorated with incremental change data: 553 | ```javascript 554 | [...{ 555 | key: 'tag5' 556 | value: { 557 | sum: 5 558 | sumChange: 7, 559 | sumChangeFromStart: 0, 560 | sumChangeFromEnd: 30, 561 | otherColumn: { 562 | avgChange: 4 563 | avgChangeFromStart: -4 564 | avgChangeFromEnd: -20 565 | } 566 | } 567 | }...] 568 | ``` 569 | 570 | 571 |

.post(callback) #

572 | 573 | - Description 574 | - Use a custom callback function to perform your own post aggregations. 575 | - Parameters 576 | - `callback` - the callback function to execute. It accepts the following parameters: 577 | - `query` - the new query object. A fresh reference (or copy, if the parent is locked) is located at `query.data`. It is highly discouraged to change any other property on this object 578 | - `parentQuery` - the parent query. 579 | - You may optionally return a promise-like value for asynchronous processing 580 | ```javascript 581 | .post(function(query, parentQuery){ 582 | query.data[0].key = 'newKeyName' 583 | return Promise.resolve(doSomethingSpecial(query.data)) 584 | }) 585 | ``` 586 | - Returns 587 | - `promise` resolved with 588 | - **query instance** 589 | 590 | 591 | 592 | 593 | 594 | 595 | 596 | 597 | 598 | 599 | 600 | 601 | 602 | 603 |

Pro Tips #

604 | 605 | #### No Arrays Necessary 606 | Don’t want to use arrays in your aggregations? No problem, because this: 607 | 608 | ```javascript 609 | .then(function(universe){ 610 | universe.query({ 611 | select: { 612 | $sum: { 613 | $sum: [ 614 | {$max: ['tip', 'total']}, 615 | {$min: ['quantity', 'total']} 616 | ] 617 | }, 618 | } 619 | }) 620 | }) 621 | ``` 622 | … is now easier written like this: 623 | 624 | ```javascript 625 | .then(function(universe){ 626 | universe.query({ 627 | select: { 628 | $sum: { 629 | $sum: { 630 | $max: ['tip', 'total'], 631 | $min: ['quantity', 'total'] 632 | } 633 | }, 634 | } 635 | }) 636 | }) 637 | ``` 638 | 639 | #### No Objects Necessary, either! 640 | What’s that? Don’t like the verbosity of objects or arrays? Use the new string syntax! 641 | 642 | ```javascript 643 | .then(function(universe){ 644 | universe.query({ 645 | select: { 646 | $sum: '$sum($max(tip,total), $min(quantity,total))' 647 | } 648 | }) 649 | }) 650 | ``` 651 | 652 | #### Pre-compile Columns 653 | 654 | Pro-Tip: You can also **pre-compile** column indices before querying. Otherwise, ad-hoc indices are created and managed automagically for you anyway. 655 | 656 | ```javascript 657 | .then(function(myUniverse){ 658 | return myUniverse.column('a') 659 | return myUniverse.column(['a', 'b', 'c']) 660 | return myUniverse.column({ 661 | key: 'd', 662 | type: 'string' // override automatic type detection 663 | }) 664 | }) 665 | ``` 666 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "universe", 3 | "version": "0.8.1", 4 | "description": "The fastest way to query and explore multivariate datasets", 5 | "main": "src/universe.js", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "files": ["src", "universe.*"], 10 | "dependencies": { 11 | "crossfilter2": "1.4.1", 12 | "reductio": "^0.6.2" 13 | }, 14 | "devDependencies": { 15 | "ava": "^0.15.2", 16 | "browserify": "^13.0.0", 17 | "browserify-shim": "^3.8.12", 18 | "uglify-js": "^2.7.0" 19 | }, 20 | "scripts": { 21 | "lint": "eslint src", 22 | "test": "yarn lint && ava --verbose", 23 | "browserify": "browserify ./src/universe.js -d -s universe -o universe.js", 24 | "min": "uglifyjs universe.js -o universe.min.js", 25 | "build": "npm run browserify && npm run min && echo 'Done building.'", 26 | "watch": "onchange 'src/**' -i -w -- npm run build", 27 | "prepublish": "npm run build", 28 | "postpublish": "git push --tags" 29 | }, 30 | "repository": { 31 | "type": "git", 32 | "url": "https://github.com/crossfilter/universe.git" 33 | }, 34 | "keywords": [ 35 | "crossfilter", 36 | "query", 37 | "multivariate", 38 | "datavis", 39 | "filtering", 40 | "data" 41 | ], 42 | "author": "Tanner Linsley", 43 | "license": "Apache-2.0", 44 | "bugs": { 45 | "url": "https://github.com/crossfilter/universe/issues" 46 | }, 47 | "homepage": "https://github.com/crossfilter/universe" 48 | } 49 | -------------------------------------------------------------------------------- /src/aggregation.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var _ = require('./lodash') 4 | 5 | var aggregators = { 6 | // Collections 7 | $sum: $sum, 8 | $avg: $avg, 9 | $max: $max, 10 | $min: $min, 11 | 12 | // Pickers 13 | $count: $count, 14 | $first: $first, 15 | $last: $last, 16 | $get: $get, 17 | $nth: $get, // nth is same as using a get 18 | $nthLast: $nthLast, 19 | $nthPct: $nthPct, 20 | $map: $map, 21 | } 22 | 23 | module.exports = { 24 | makeValueAccessor: makeValueAccessor, 25 | aggregators: aggregators, 26 | extractKeyValOrArray: extractKeyValOrArray, 27 | parseAggregatorParams: parseAggregatorParams, 28 | } 29 | // This is used to build aggregation stacks for sub-reductio 30 | // aggregations, or plucking values for use in filters from the data 31 | function makeValueAccessor(obj) { 32 | if (typeof obj === 'string') { 33 | if (isStringSyntax(obj)) { 34 | obj = convertAggregatorString(obj) 35 | } else { 36 | // Must be a column key. Return an identity accessor 37 | return obj 38 | } 39 | } 40 | // Must be a column index. Return an identity accessor 41 | if (typeof obj === 'number') { 42 | return obj 43 | } 44 | // If it's an object, we need to build a custom value accessor function 45 | if (_.isObject(obj)) { 46 | return make() 47 | } 48 | 49 | function make() { 50 | var stack = makeSubAggregationFunction(obj) 51 | return function topStack(d) { 52 | return stack(d) 53 | } 54 | } 55 | } 56 | 57 | // A recursive function that walks the aggregation stack and returns 58 | // a function. The returned function, when called, will recursively invoke 59 | // with the properties from the previous stack in reverse order 60 | function makeSubAggregationFunction(obj) { 61 | // If its an object, either unwrap all of the properties as an 62 | // array of keyValues, or unwrap the first keyValue set as an object 63 | obj = _.isObject(obj) ? extractKeyValOrArray(obj) : obj 64 | 65 | // Detect strings 66 | if (_.isString(obj)) { 67 | // If begins with a $, then we need to convert it over to a regular query object and analyze it again 68 | if (isStringSyntax(obj)) { 69 | return makeSubAggregationFunction(convertAggregatorString(obj)) 70 | } 71 | // If normal string, then just return a an itentity accessor 72 | return function identity(d) { 73 | return d[obj] 74 | } 75 | } 76 | 77 | // If an array, recurse into each item and return as a map 78 | if (_.isArray(obj)) { 79 | var subStack = _.map(obj, makeSubAggregationFunction) 80 | return function getSubStack(d) { 81 | return subStack.map(function(s) { 82 | return s(d) 83 | }) 84 | } 85 | } 86 | 87 | // If object, find the aggregation, and recurse into the value 88 | if (obj.key) { 89 | if (aggregators[obj.key]) { 90 | var subAggregationFunction = makeSubAggregationFunction(obj.value) 91 | return function getAggregation(d) { 92 | return aggregators[obj.key](subAggregationFunction(d)) 93 | } 94 | } 95 | console.error('Could not find aggregration method', obj) 96 | } 97 | 98 | return [] 99 | } 100 | 101 | function extractKeyValOrArray(obj) { 102 | var keyVal 103 | var values = [] 104 | for (var key in obj) { 105 | if ({}.hasOwnProperty.call(obj, key)) { 106 | keyVal = { 107 | key: key, 108 | value: obj[key], 109 | } 110 | var subObj = {} 111 | subObj[key] = obj[key] 112 | values.push(subObj) 113 | } 114 | } 115 | return values.length > 1 ? values : keyVal 116 | } 117 | 118 | function isStringSyntax(str) { 119 | return ['$', '('].indexOf(str.charAt(0)) > -1 120 | } 121 | 122 | function parseAggregatorParams(keyString) { 123 | var params = [] 124 | var p1 = keyString.indexOf('(') 125 | var p2 = keyString.indexOf(')') 126 | var key = p1 > -1 ? keyString.substring(0, p1) : keyString 127 | if (!aggregators[key]) { 128 | return false 129 | } 130 | if (p1 > -1 && p2 > -1 && p2 > p1) { 131 | params = keyString.substring(p1 + 1, p2).split(',') 132 | } 133 | 134 | return { 135 | aggregator: aggregators[key], 136 | params: params, 137 | } 138 | } 139 | 140 | function convertAggregatorString(keyString) { 141 | // var obj = {} // obj is defined but not used 142 | 143 | // 1. unwrap top parentheses 144 | // 2. detect arrays 145 | 146 | // parentheses 147 | var outerParens = /\((.+)\)/g 148 | // var innerParens = /\(([^\(\)]+)\)/g // innerParens is defined but not used 149 | // comma not in () 150 | var hasComma = /(?:\([^\(\)]*\))|(,)/g 151 | 152 | return JSON.parse('{' + unwrapParensAndCommas(keyString) + '}') 153 | 154 | function unwrapParensAndCommas(str) { 155 | str = str.replace(' ', '') 156 | return ( 157 | '"' + 158 | str.replace(outerParens, function(p, pr) { 159 | if (hasComma.test(pr)) { 160 | if (pr.charAt(0) === '$') { 161 | return ( 162 | '":{"' + 163 | pr.replace(hasComma, function(p2 /* , pr2 */) { 164 | if (p2 === ',') { 165 | return ',"' 166 | } 167 | return unwrapParensAndCommas(p2).trim() 168 | }) + 169 | '}' 170 | ) 171 | } 172 | return ( 173 | ':["' + 174 | pr.replace( 175 | hasComma, 176 | function(/* p2 , pr2 */) { 177 | return '","' 178 | } 179 | ) + 180 | '"]' 181 | ) 182 | } 183 | }) 184 | ) 185 | } 186 | } 187 | 188 | // Collection Aggregators 189 | 190 | function $sum(children) { 191 | return children.reduce(function(a, b) { 192 | return a + b 193 | }, 0) 194 | } 195 | 196 | function $avg(children) { 197 | return ( 198 | children.reduce(function(a, b) { 199 | return a + b 200 | }, 0) / children.length 201 | ) 202 | } 203 | 204 | function $max(children) { 205 | return Math.max.apply(null, children) 206 | } 207 | 208 | function $min(children) { 209 | return Math.min.apply(null, children) 210 | } 211 | 212 | function $count(children) { 213 | return children.length 214 | } 215 | 216 | /* function $med(children) { // $med is defined but not used 217 | children.sort(function(a, b) { 218 | return a - b 219 | }) 220 | var half = Math.floor(children.length / 2) 221 | if (children.length % 2) 222 | return children[half] 223 | else 224 | return (children[half - 1] + children[half]) / 2.0 225 | } */ 226 | 227 | function $first(children) { 228 | return children[0] 229 | } 230 | 231 | function $last(children) { 232 | return children[children.length - 1] 233 | } 234 | 235 | function $get(children, n) { 236 | return children[n] 237 | } 238 | 239 | function $nthLast(children, n) { 240 | return children[children.length - n] 241 | } 242 | 243 | function $nthPct(children, n) { 244 | return children[Math.round(children.length * (n / 100))] 245 | } 246 | 247 | function $map(children, n) { 248 | return children.map(function(d) { 249 | return d[n] 250 | }) 251 | } 252 | -------------------------------------------------------------------------------- /src/clear.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var _ = require('./lodash') 4 | 5 | module.exports = function(service) { 6 | return function clear(def) { 7 | // Clear a single or multiple column definitions 8 | if (def) { 9 | def = _.isArray(def) ? def : [def] 10 | } 11 | 12 | if (!def) { 13 | // Clear all of the column defenitions 14 | return Promise.all( 15 | _.map(service.columns, disposeColumn) 16 | ).then(function() { 17 | service.columns = [] 18 | return service 19 | }) 20 | } 21 | 22 | return Promise.all( 23 | _.map(def, function(d) { 24 | if (_.isObject(d)) { 25 | d = d.key 26 | } 27 | // Clear the column 28 | var column = _.remove(service.columns, function(c) { 29 | if (_.isArray(d)) { 30 | return !_.xor(c.key, d).length 31 | } 32 | if (c.key === d) { 33 | if (c.dynamicReference) { 34 | return false 35 | } 36 | return true 37 | } 38 | })[0] 39 | 40 | if (!column) { 41 | // console.info('Attempted to clear a column that is required for another query!', c) 42 | return 43 | } 44 | 45 | disposeColumn(column) 46 | }) 47 | ).then(function() { 48 | return service 49 | }) 50 | 51 | function disposeColumn(column) { 52 | var disposalActions = [] 53 | // Dispose the dimension 54 | if (column.removeListeners) { 55 | disposalActions = _.map(column.removeListeners, function(listener) { 56 | return Promise.resolve(listener()) 57 | }) 58 | } 59 | var filterKey = column.key 60 | if (column.complex === 'array') { 61 | filterKey = JSON.stringify(column.key) 62 | } 63 | if (column.complex === 'function') { 64 | filterKey = column.key.toString() 65 | } 66 | delete service.filters[filterKey] 67 | if (column.dimension) { 68 | disposalActions.push(Promise.resolve(column.dimension.dispose())) 69 | } 70 | return Promise.all(disposalActions) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/column.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var _ = require('./lodash') 4 | 5 | module.exports = function (service) { 6 | var dimension = require('./dimension')(service) 7 | 8 | var columnFunc = column 9 | columnFunc.find = findColumn 10 | 11 | return columnFunc 12 | 13 | function column(def) { 14 | // Support groupAll dimension 15 | if (_.isUndefined(def)) { 16 | def = true 17 | } 18 | 19 | // Always deal in bulk. Like Costco! 20 | if (!_.isArray(def)) { 21 | def = [def] 22 | } 23 | 24 | // Mapp all column creation, wait for all to settle, then return the instance 25 | return Promise.all(_.map(def, makeColumn)) 26 | .then(function () { 27 | return service 28 | }) 29 | } 30 | 31 | function findColumn(d) { 32 | return _.find(service.columns, function (c) { 33 | if (_.isArray(d)) { 34 | return !_.xor(c.key, d).length 35 | } 36 | return c.key === d 37 | }) 38 | } 39 | 40 | function getType(d) { 41 | if (_.isNumber(d)) { 42 | return 'number' 43 | } 44 | if (_.isBoolean(d)) { 45 | return 'bool' 46 | } 47 | if (_.isArray(d)) { 48 | return 'array' 49 | } 50 | if (_.isObject(d)) { 51 | return 'object' 52 | } 53 | return 'string' 54 | } 55 | 56 | function makeColumn(d) { 57 | var column = _.isObject(d) ? d : { 58 | key: d, 59 | } 60 | 61 | var existing = findColumn(column.key) 62 | 63 | if (existing) { 64 | existing.temporary = false 65 | if (existing.dynamicReference) { 66 | existing.dynamicReference = false 67 | } 68 | return existing.promise 69 | .then(function () { 70 | return service 71 | }) 72 | } 73 | 74 | // for storing info about queries and post aggregations 75 | column.queries = [] 76 | service.columns.push(column) 77 | 78 | column.promise = new Promise(function (resolve, reject) { 79 | try { 80 | resolve(service.cf.all()) 81 | } catch (err) { 82 | reject(err) 83 | } 84 | }) 85 | .then(function (all) { 86 | var sample 87 | 88 | // Complex column Keys 89 | if (_.isFunction(column.key)) { 90 | column.complex = 'function' 91 | sample = column.key(all[0]) 92 | } else if (_.isString(column.key) && (column.key.indexOf('.') > -1 || column.key.indexOf('[') > -1)) { 93 | column.complex = 'string' 94 | sample = _.get(all[0], column.key) 95 | } else if (_.isArray(column.key)) { 96 | column.complex = 'array' 97 | sample = _.values(_.pick(all[0], column.key)) 98 | if (sample.length !== column.key.length) { 99 | throw new Error('Column key does not exist in data!', column.key) 100 | } 101 | } else { 102 | sample = all[0][column.key] 103 | } 104 | 105 | // Index Column 106 | if (!column.complex && column.key !== true && typeof sample === 'undefined') { 107 | throw new Error('Column key does not exist in data!', column.key) 108 | } 109 | 110 | // If the column exists, let's at least make sure it's marked 111 | // as permanent. There is a slight chance it exists because 112 | // of a filter, and the user decides to make it permanent 113 | 114 | if (column.key === true) { 115 | column.type = 'all' 116 | } else if (column.complex) { 117 | column.type = 'complex' 118 | } else if (column.array) { 119 | column.type = 'array' 120 | } else { 121 | column.type = getType(sample) 122 | } 123 | 124 | return dimension.make(column.key, column.type, column.complex) 125 | }) 126 | .then(function (dim) { 127 | column.dimension = dim 128 | column.filterCount = 0 129 | var stopListeningForData = service.onDataChange(buildColumnKeys) 130 | column.removeListeners = [stopListeningForData] 131 | 132 | return buildColumnKeys() 133 | 134 | // Build the columnKeys 135 | function buildColumnKeys(changes) { 136 | if (column.key === true) { 137 | return Promise.resolve() 138 | } 139 | 140 | var accessor = dimension.makeAccessor(column.key, column.complex) 141 | column.values = column.values || [] 142 | 143 | return new Promise(function (resolve, reject) { 144 | try { 145 | if (changes && changes.added) { 146 | resolve(changes.added) 147 | } else { 148 | resolve(column.dimension.bottom(Infinity)) 149 | } 150 | } catch (err) { 151 | reject(err) 152 | } 153 | }) 154 | .then(function (rows) { 155 | var newValues 156 | if (column.complex === 'string' || column.complex === 'function') { 157 | newValues = _.map(rows, accessor) 158 | // console.log(rows, accessor.toString(), newValues) 159 | } else if (column.type === 'array') { 160 | newValues = _.flatten(_.map(rows, accessor)) 161 | } else { 162 | newValues = _.map(rows, accessor) 163 | } 164 | column.values = _.uniq(column.values.concat(newValues)) 165 | }) 166 | } 167 | }) 168 | 169 | return column.promise 170 | .then(function () { 171 | return service 172 | }) 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/crossfilter.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var crossfilter = require('crossfilter2') 4 | 5 | var _ = require('./lodash') 6 | 7 | module.exports = function (service) { 8 | return { 9 | build: build, 10 | generateColumns: generateColumns, 11 | add: add, 12 | remove: remove, 13 | } 14 | 15 | function build(c) { 16 | if (_.isArray(c)) { 17 | // This allows support for crossfilter async 18 | return Promise.resolve(crossfilter(c)) 19 | } 20 | if (!c || typeof c.dimension !== 'function') { 21 | return Promise.reject(new Error('No Crossfilter data or instance found!')) 22 | } 23 | return Promise.resolve(c) 24 | } 25 | 26 | function generateColumns(data) { 27 | if (!service.options.generatedColumns) { 28 | return data 29 | } 30 | return _.map(data, function (d/* , i */) { 31 | _.forEach(service.options.generatedColumns, function (val, key) { 32 | d[key] = val(d) 33 | }) 34 | return d 35 | }) 36 | } 37 | 38 | function add(data) { 39 | data = generateColumns(data) 40 | return new Promise(function (resolve, reject) { 41 | try { 42 | resolve(service.cf.add(data)) 43 | } catch (err) { 44 | reject(err) 45 | } 46 | }) 47 | .then(function () { 48 | return _.map(service.dataListeners, function (listener) { 49 | return function () { 50 | return listener({ 51 | added: data, 52 | }) 53 | } 54 | }).reduce(function(promise, data) { 55 | return promise.then(data) 56 | }, Promise.resolve(true)) 57 | }) 58 | .then(function () { 59 | return service 60 | }) 61 | } 62 | 63 | function remove() { 64 | return new Promise(function (resolve, reject) { 65 | try { 66 | resolve(service.cf.remove()) 67 | } catch (err) { 68 | reject(err) 69 | } 70 | }) 71 | .then(function () { 72 | return service 73 | }) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/destroy.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | // var _ = require('./lodash') // _ is defined but never used 4 | 5 | module.exports = function (service) { 6 | return function destroy() { 7 | return service.clear() 8 | .then(function () { 9 | service.cf.dataListeners = [] 10 | service.cf.filterListeners = [] 11 | return Promise.resolve(service.cf.remove()) 12 | }) 13 | .then(function () { 14 | return service 15 | }) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/dimension.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var _ = require('./lodash') 4 | 5 | module.exports = function (service) { 6 | return { 7 | make: make, 8 | makeAccessor: makeAccessor, 9 | } 10 | 11 | function make(key, type, complex) { 12 | var accessor = makeAccessor(key, complex) 13 | // Promise.resolve will handle promises or non promises, so 14 | // this crossfilter async is supported if present 15 | return Promise.resolve(service.cf.dimension(accessor, type === 'array')) 16 | } 17 | 18 | function makeAccessor(key, complex) { 19 | var accessorFunction 20 | 21 | if (complex === 'string') { 22 | accessorFunction = function (d) { 23 | return _.get(d, key) 24 | } 25 | } else if (complex === 'function') { 26 | accessorFunction = key 27 | } else if (complex === 'array') { 28 | var arrayString = _.map(key, function (k) { 29 | return 'd[\'' + k + '\']' 30 | }) 31 | accessorFunction = new Function('d', String('return ' + JSON.stringify(arrayString).replace(/"/g, ''))) // eslint-disable-line no-new-func 32 | } else { 33 | accessorFunction = 34 | // Index Dimension 35 | key === true ? function accessor(d, i) { 36 | return i 37 | } : 38 | // Value Accessor Dimension 39 | function (d) { 40 | return d[key] 41 | } 42 | } 43 | return accessorFunction 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/expressions.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | // var moment = require('moment') 4 | 5 | module.exports = { 6 | // Getters 7 | $field: $field, 8 | // Booleans 9 | $and: $and, 10 | $or: $or, 11 | $not: $not, 12 | 13 | // Expressions 14 | $eq: $eq, 15 | $gt: $gt, 16 | $gte: $gte, 17 | $lt: $lt, 18 | $lte: $lte, 19 | $ne: $ne, 20 | $type: $type, 21 | 22 | // Array Expressions 23 | $in: $in, 24 | $nin: $nin, 25 | $contains: $contains, 26 | $excludes: $excludes, 27 | $size: $size, 28 | } 29 | 30 | // Getters 31 | function $field(d, child) { 32 | return d[child] 33 | } 34 | 35 | // Operators 36 | 37 | function $and(d, child) { 38 | child = child(d) 39 | for (var i = 0; i < child.length; i++) { 40 | if (!child[i]) { 41 | return false 42 | } 43 | } 44 | return true 45 | } 46 | 47 | function $or(d, child) { 48 | child = child(d) 49 | for (var i = 0; i < child.length; i++) { 50 | if (child[i]) { 51 | return true 52 | } 53 | } 54 | return false 55 | } 56 | 57 | function $not(d, child) { 58 | child = child(d) 59 | for (var i = 0; i < child.length; i++) { 60 | if (child[i]) { 61 | return false 62 | } 63 | } 64 | return true 65 | } 66 | 67 | // Expressions 68 | 69 | function $eq(d, child) { 70 | return d === child() 71 | } 72 | 73 | function $gt(d, child) { 74 | return d > child() 75 | } 76 | 77 | function $gte(d, child) { 78 | return d >= child() 79 | } 80 | 81 | function $lt(d, child) { 82 | return d < child() 83 | } 84 | 85 | function $lte(d, child) { 86 | return d <= child() 87 | } 88 | 89 | function $ne(d, child) { 90 | return d !== child() 91 | } 92 | 93 | function $type(d, child) { 94 | return typeof d === child() 95 | } 96 | 97 | // Array Expressions 98 | 99 | function $in(d, child) { 100 | return d.indexOf(child()) > -1 101 | } 102 | 103 | function $nin(d, child) { 104 | return d.indexOf(child()) === -1 105 | } 106 | 107 | function $contains(d, child) { 108 | return child().indexOf(d) > -1 109 | } 110 | 111 | function $excludes(d, child) { 112 | return child().indexOf(d) === -1 113 | } 114 | 115 | function $size(d, child) { 116 | return d.length === child() 117 | } 118 | -------------------------------------------------------------------------------- /src/filters.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var _ = require('./lodash') 4 | 5 | var expressions = require('./expressions') 6 | var aggregation = require('./aggregation') 7 | 8 | module.exports = function (service) { 9 | return { 10 | filter: filter, 11 | filterAll: filterAll, 12 | applyFilters: applyFilters, 13 | makeFunction: makeFunction, 14 | scanForDynamicFilters: scanForDynamicFilters, 15 | } 16 | 17 | function filter(column, fil, isRange, replace) { 18 | return getColumn(column) 19 | .then(function (column) { 20 | // Clone a copy of the new filters 21 | var newFilters = _.assign({}, service.filters) 22 | // Here we use the registered column key despite the filter key passed, just in case the filter key's ordering is ordered differently :) 23 | var filterKey = column.key 24 | if (column.complex === 'array') { 25 | filterKey = JSON.stringify(column.key) 26 | } 27 | if (column.complex === 'function') { 28 | filterKey = column.key.toString() 29 | } 30 | // Build the filter object 31 | newFilters[filterKey] = buildFilterObject(fil, isRange, replace) 32 | 33 | return applyFilters(newFilters) 34 | }) 35 | } 36 | 37 | function getColumn(column) { 38 | var exists = service.column.find(column) 39 | // If the filters dimension doesn't exist yet, try and create it 40 | return new Promise(function (resolve, reject) { 41 | try { 42 | if (!exists) { 43 | return resolve(service.column({ 44 | key: column, 45 | temporary: true, 46 | }) 47 | .then(function () { 48 | // It was able to be created, so retrieve and return it 49 | return service.column.find(column) 50 | }) 51 | ) 52 | } else { 53 | // It exists, so just return what we found 54 | resolve(exists) 55 | } 56 | } catch (err) { 57 | reject(err) 58 | } 59 | }) 60 | } 61 | 62 | function filterAll(fils) { 63 | // If empty, remove all filters 64 | if (!fils) { 65 | service.columns.forEach(function (col) { 66 | col.dimension.filterAll() 67 | }) 68 | return applyFilters({}) 69 | } 70 | 71 | // Clone a copy for the new filters 72 | var newFilters = _.assign({}, service.filters) 73 | 74 | var ds = _.map(fils, function (fil) { 75 | return getColumn(fil.column) 76 | .then(function (column) { 77 | // Here we use the registered column key despite the filter key passed, just in case the filter key's ordering is ordered differently :) 78 | var filterKey = column.complex ? JSON.stringify(column.key) : column.key 79 | // Build the filter object 80 | newFilters[filterKey] = buildFilterObject(fil.value, fil.isRange, fil.replace) 81 | }) 82 | }) 83 | 84 | return Promise.all(ds) 85 | .then(function () { 86 | return applyFilters(newFilters) 87 | }) 88 | } 89 | 90 | function buildFilterObject(fil, isRange, replace) { 91 | if (_.isUndefined(fil)) { 92 | return false 93 | } 94 | if (_.isFunction(fil)) { 95 | return { 96 | value: fil, 97 | function: fil, 98 | replace: true, 99 | type: 'function', 100 | } 101 | } 102 | if (_.isObject(fil)) { 103 | return { 104 | value: fil, 105 | function: makeFunction(fil), 106 | replace: true, 107 | type: 'function', 108 | } 109 | } 110 | if (_.isArray(fil)) { 111 | return { 112 | value: fil, 113 | replace: isRange || replace, 114 | type: isRange ? 'range' : 'inclusive', 115 | } 116 | } 117 | return { 118 | value: fil, 119 | replace: replace, 120 | type: 'exact', 121 | } 122 | } 123 | 124 | function applyFilters(newFilters) { 125 | var ds = _.map(newFilters, function (fil, i) { 126 | var existing = service.filters[i] 127 | // Filters are the same, so no change is needed on this column 128 | if (fil === existing) { 129 | return Promise.resolve() 130 | } 131 | var column 132 | // Retrieve complex columns by decoding the column key as json 133 | if (i.charAt(0) === '[') { 134 | column = service.column.find(JSON.parse(i)) 135 | } else { 136 | // Retrieve the column normally 137 | column = service.column.find(i) 138 | } 139 | 140 | // Toggling a filter value is a bit different from replacing them 141 | if (fil && existing && !fil.replace) { 142 | newFilters[i] = fil = toggleFilters(fil, existing) 143 | } 144 | 145 | // If no filter, remove everything from the dimension 146 | if (!fil) { 147 | return Promise.resolve(column.dimension.filterAll()) 148 | } 149 | if (fil.type === 'exact') { 150 | return Promise.resolve(column.dimension.filterExact(fil.value)) 151 | } 152 | if (fil.type === 'range') { 153 | return Promise.resolve(column.dimension.filterRange(fil.value)) 154 | } 155 | if (fil.type === 'inclusive') { 156 | return Promise.resolve(column.dimension.filterFunction(function (d) { 157 | return fil.value.indexOf(d) > -1 158 | })) 159 | } 160 | if (fil.type === 'function') { 161 | return Promise.resolve(column.dimension.filterFunction(fil.function)) 162 | } 163 | // By default if something craps up, just remove all filters 164 | return Promise.resolve(column.dimension.filterAll()) 165 | }) 166 | 167 | return Promise.all(ds) 168 | .then(function () { 169 | // Save the new filters satate 170 | service.filters = newFilters 171 | 172 | // Pluck and remove falsey filters from the mix 173 | var tryRemoval = [] 174 | _.forEach(service.filters, function (val, key) { 175 | if (!val) { 176 | tryRemoval.push({ 177 | key: key, 178 | val: val, 179 | }) 180 | delete service.filters[key] 181 | } 182 | }) 183 | 184 | // If any of those filters are the last dependency for the column, then remove the column 185 | return Promise.all(_.map(tryRemoval, function (v) { 186 | var column = service.column.find((v.key.charAt(0) === '[') ? JSON.parse(v.key) : v.key) 187 | if (column.temporary && !column.dynamicReference) { 188 | return service.clear(column.key) 189 | } 190 | })) 191 | }) 192 | .then(function () { 193 | // Call the filterListeners and wait for their return 194 | return Promise.all(_.map(service.filterListeners, function (listener) { 195 | return listener() 196 | })) 197 | }) 198 | .then(function () { 199 | return service 200 | }) 201 | } 202 | 203 | function toggleFilters(fil, existing) { 204 | // Exact from Inclusive 205 | if (fil.type === 'exact' && existing.type === 'inclusive') { 206 | fil.value = _.xor([fil.value], existing.value) 207 | } else if (fil.type === 'inclusive' && existing.type === 'exact') { // Inclusive from Exact 208 | fil.value = _.xor(fil.value, [existing.value]) 209 | } else if (fil.type === 'inclusive' && existing.type === 'inclusive') { // Inclusive / Inclusive Merge 210 | fil.value = _.xor(fil.value, existing.value) 211 | } else if (fil.type === 'exact' && existing.type === 'exact') { // Exact / Exact 212 | // If the values are the same, remove the filter entirely 213 | if (fil.value === existing.value) { 214 | return false 215 | } 216 | // They they are different, make an array 217 | fil.value = [fil.value, existing.value] 218 | } 219 | 220 | // Set the new type based on the merged values 221 | if (!fil.value.length) { 222 | fil = false 223 | } else if (fil.value.length === 1) { 224 | fil.type = 'exact' 225 | fil.value = fil.value[0] 226 | } else { 227 | fil.type = 'inclusive' 228 | } 229 | 230 | return fil 231 | } 232 | 233 | function scanForDynamicFilters(query) { 234 | // Here we check to see if there are any relative references to the raw data 235 | // being used in the filter. If so, we need to build those dimensions and keep 236 | // them updated so the filters can be rebuilt if needed 237 | // The supported keys right now are: $column, $data 238 | var columns = [] 239 | walk(query.filter) 240 | return columns 241 | 242 | function walk(obj) { 243 | _.forEach(obj, function (val, key) { 244 | // find the data references, if any 245 | var ref = findDataReferences(val, key) 246 | if (ref) { 247 | columns.push(ref) 248 | } 249 | // if it's a string 250 | if (_.isString(val)) { 251 | ref = findDataReferences(null, val) 252 | if (ref) { 253 | columns.push(ref) 254 | } 255 | } 256 | // If it's another object, keep looking 257 | if (_.isObject(val)) { 258 | walk(val) 259 | } 260 | }) 261 | } 262 | } 263 | 264 | function findDataReferences(val, key) { 265 | // look for the $data string as a value 266 | if (key === '$data') { 267 | return true 268 | } 269 | 270 | // look for the $column key and it's value as a string 271 | if (key && key === '$column') { 272 | if (_.isString(val)) { 273 | return val 274 | } 275 | console.warn('The value for filter "$column" must be a valid column key', val) 276 | return false 277 | } 278 | } 279 | 280 | function makeFunction(obj, isAggregation) { 281 | var subGetters 282 | 283 | // Detect raw $data reference 284 | if (_.isString(obj)) { 285 | var dataRef = findDataReferences(null, obj) 286 | if (dataRef) { 287 | var data = service.cf.all() 288 | return function () { 289 | return data 290 | } 291 | } 292 | } 293 | 294 | if (_.isString(obj) || _.isNumber(obj) || _.isBoolean(obj)) { 295 | return function (d) { 296 | if (typeof d === 'undefined') { 297 | return obj 298 | } 299 | return expressions.$eq(d, function () { 300 | return obj 301 | }) 302 | } 303 | } 304 | 305 | // If an array, recurse into each item and return as a map 306 | if (_.isArray(obj)) { 307 | subGetters = _.map(obj, function (o) { 308 | return makeFunction(o, isAggregation) 309 | }) 310 | return function (d) { 311 | return subGetters.map(function (s) { 312 | return s(d) 313 | }) 314 | } 315 | } 316 | 317 | // If object, return a recursion function that itself, returns the results of all of the object keys 318 | if (_.isObject(obj)) { 319 | subGetters = _.map(obj, function (val, key) { 320 | // Get the child 321 | var getSub = makeFunction(val, isAggregation) 322 | 323 | // Detect raw $column references 324 | var dataRef = findDataReferences(val, key) 325 | if (dataRef) { 326 | var column = service.column.find(dataRef) 327 | var data = column.values 328 | return function () { 329 | return data 330 | } 331 | } 332 | 333 | // If expression, pass the parentValue and the subGetter 334 | if (expressions[key]) { 335 | return function (d) { 336 | return expressions[key](d, getSub) 337 | } 338 | } 339 | 340 | var aggregatorObj = aggregation.parseAggregatorParams(key) 341 | if (aggregatorObj) { 342 | // Make sure that any further operations are for aggregations 343 | // and not filters 344 | isAggregation = true 345 | // here we pass true to makeFunction which denotes that 346 | // an aggregatino chain has started and to stop using $AND 347 | getSub = makeFunction(val, isAggregation) 348 | // If it's an aggregation object, be sure to pass in the children, and then any additional params passed into the aggregation string 349 | return function () { 350 | return aggregatorObj.aggregator.apply(null, [getSub()].concat(aggregatorObj.params)) 351 | } 352 | } 353 | 354 | // It must be a string then. Pluck that string key from parent, and pass it as the new value to the subGetter 355 | return function (d) { 356 | d = d[key] 357 | return getSub(d, getSub) 358 | } 359 | }) 360 | 361 | // All object expressions are basically AND's 362 | // Return AND with a map of the subGetters 363 | if (isAggregation) { 364 | if (subGetters.length === 1) { 365 | return function (d) { 366 | return subGetters[0](d) 367 | } 368 | } 369 | return function (d) { 370 | return _.map(subGetters, function (getSub) { 371 | return getSub(d) 372 | }) 373 | } 374 | } 375 | return function (d) { 376 | return expressions.$and(d, function (d) { 377 | return _.map(subGetters, function (getSub) { 378 | return getSub(d) 379 | }) 380 | }) 381 | } 382 | } 383 | 384 | console.log('no expression found for ', obj) 385 | return false 386 | } 387 | } 388 | -------------------------------------------------------------------------------- /src/lodash.js: -------------------------------------------------------------------------------- 1 | /* eslint no-prototype-builtins: 0 */ 2 | 'use strict' 3 | 4 | module.exports = { 5 | assign: assign, 6 | find: find, 7 | remove: remove, 8 | isArray: isArray, 9 | isObject: isObject, 10 | isBoolean: isBoolean, 11 | isString: isString, 12 | isNumber: isNumber, 13 | isFunction: isFunction, 14 | get: get, 15 | set: set, 16 | map: map, 17 | keys: keys, 18 | sortBy: sortBy, 19 | forEach: forEach, 20 | isUndefined: isUndefined, 21 | pick: pick, 22 | xor: xor, 23 | clone: clone, 24 | isEqual: isEqual, 25 | replaceArray: replaceArray, 26 | uniq: uniq, 27 | flatten: flatten, 28 | sort: sort, 29 | values: values, 30 | recurseObject: recurseObject, 31 | } 32 | 33 | function assign(out) { 34 | out = out || {} 35 | for (var i = 1; i < arguments.length; i++) { 36 | if (!arguments[i]) { 37 | continue 38 | } 39 | for (var key in arguments[i]) { 40 | if (arguments[i].hasOwnProperty(key)) { 41 | out[key] = arguments[i][key] 42 | } 43 | } 44 | } 45 | return out 46 | } 47 | 48 | function find(a, b) { 49 | return a.find(b) 50 | } 51 | 52 | function remove(a, b) { 53 | return a.filter(function (o, i) { 54 | var r = b(o) 55 | if (r) { 56 | a.splice(i, 1) 57 | return true 58 | } 59 | return false 60 | }) 61 | } 62 | 63 | function isArray(a) { 64 | return Array.isArray(a) 65 | } 66 | 67 | function isObject(d) { 68 | return typeof d === 'object' && !isArray(d) 69 | } 70 | 71 | function isBoolean(d) { 72 | return typeof d === 'boolean' 73 | } 74 | 75 | function isString(d) { 76 | return typeof d === 'string' 77 | } 78 | 79 | function isNumber(d) { 80 | return typeof d === 'number' 81 | } 82 | 83 | function isFunction(a) { 84 | return typeof a === 'function' 85 | } 86 | 87 | function get(a, b) { 88 | if (isArray(b)) { 89 | b = b.join('.') 90 | } 91 | return b 92 | .replace('[', '.').replace(']', '') 93 | .split('.') 94 | .reduce( 95 | function (obj, property) { 96 | return obj[property] 97 | }, a 98 | ) 99 | } 100 | 101 | function set(obj, prop, value) { 102 | if (typeof prop === 'string') { 103 | prop = prop 104 | .replace('[', '.').replace(']', '') 105 | .split('.') 106 | } 107 | if (prop.length > 1) { 108 | var e = prop.shift() 109 | assign(obj[e] = 110 | Object.prototype.toString.call(obj[e]) === '[object Object]' ? obj[e] : {}, 111 | prop, 112 | value) 113 | } else { 114 | obj[prop[0]] = value 115 | } 116 | } 117 | 118 | function map(a, b) { 119 | var m 120 | var key 121 | if (isFunction(b)) { 122 | if (isObject(a)) { 123 | m = [] 124 | for (key in a) { 125 | if (a.hasOwnProperty(key)) { 126 | m.push(b(a[key], key, a)) 127 | } 128 | } 129 | return m 130 | } 131 | return a.map(b) 132 | } 133 | if (isObject(a)) { 134 | m = [] 135 | for (key in a) { 136 | if (a.hasOwnProperty(key)) { 137 | m.push(a[key]) 138 | } 139 | } 140 | return m 141 | } 142 | return a.map(function (aa) { 143 | return aa[b] 144 | }) 145 | } 146 | 147 | function keys(obj) { 148 | return Object.keys(obj) 149 | } 150 | 151 | function sortBy(a, b) { 152 | if (isFunction(b)) { 153 | return a.sort(function (aa, bb) { 154 | if (b(aa) > b(bb)) { 155 | return 1 156 | } 157 | if (b(aa) < b(bb)) { 158 | return -1 159 | } 160 | // a must be equal to b 161 | return 0 162 | }) 163 | } 164 | } 165 | 166 | function forEach(a, b) { 167 | if (isObject(a)) { 168 | for (var key in a) { 169 | if (a.hasOwnProperty(key)) { 170 | b(a[key], key, a) 171 | } 172 | } 173 | return 174 | } 175 | if (isArray(a)) { 176 | return a.forEach(b) 177 | } 178 | } 179 | 180 | function isUndefined(a) { 181 | return typeof a === 'undefined' 182 | } 183 | 184 | function pick(a, b) { 185 | var c = {} 186 | forEach(b, function (bb) { 187 | if (typeof a[bb] !== 'undefined') { 188 | c[bb] = a[bb] 189 | } 190 | }) 191 | return c 192 | } 193 | 194 | function xor(a, b) { 195 | var unique = [] 196 | forEach(a, function (aa) { 197 | if (b.indexOf(aa) === -1) { 198 | return unique.push(aa) 199 | } 200 | }) 201 | forEach(b, function (bb) { 202 | if (a.indexOf(bb) === -1) { 203 | return unique.push(bb) 204 | } 205 | }) 206 | return unique 207 | } 208 | 209 | function clone(a) { 210 | return JSON.parse(JSON.stringify(a, function replacer(key, value) { 211 | if (typeof value === 'function') { 212 | return value.toString() 213 | } 214 | return value 215 | })) 216 | } 217 | 218 | function isEqual(x, y) { 219 | if ((typeof x === 'object' && x !== null) && (typeof y === 'object' && y !== null)) { 220 | if (Object.keys(x).length !== Object.keys(y).length) { 221 | return false 222 | } 223 | 224 | for (var prop in x) { 225 | if (y.hasOwnProperty(prop)) { 226 | if (!isEqual(x[prop], y[prop])) { 227 | return false 228 | } 229 | } 230 | return false 231 | } 232 | 233 | return true 234 | } else if (x !== y) { 235 | return false 236 | } 237 | return true 238 | } 239 | 240 | function replaceArray(a, b) { 241 | var al = a.length 242 | var bl = b.length 243 | if (al > bl) { 244 | a.splice(bl, al - bl) 245 | } else if (al < bl) { 246 | a.push.apply(a, new Array(bl - al)) 247 | } 248 | forEach(a, function (val, key) { 249 | a[key] = b[key] 250 | }) 251 | return a 252 | } 253 | 254 | function uniq(a) { 255 | var seen = new Set() 256 | return a.filter(function (item) { 257 | var allow = false 258 | if (!seen.has(item)) { 259 | seen.add(item) 260 | allow = true 261 | } 262 | return allow 263 | }) 264 | } 265 | 266 | function flatten(aa) { 267 | var flattened = [] 268 | for (var i = 0; i < aa.length; ++i) { 269 | var current = aa[i] 270 | for (var j = 0; j < current.length; ++j) { 271 | flattened.push(current[j]) 272 | } 273 | } 274 | return flattened 275 | } 276 | 277 | function sort(arr) { 278 | for (var i = 1; i < arr.length; i++) { 279 | var tmp = arr[i] 280 | var j = i 281 | while (arr[j - 1] > tmp) { 282 | arr[j] = arr[j - 1] 283 | --j 284 | } 285 | arr[j] = tmp 286 | } 287 | 288 | return arr 289 | } 290 | 291 | function values(a) { 292 | var values = [] 293 | for (var key in a) { 294 | if (a.hasOwnProperty(key)) { 295 | values.push(a[key]) 296 | } 297 | } 298 | return values 299 | } 300 | 301 | function recurseObject(obj, cb) { 302 | _recurseObject(obj, []) 303 | return obj 304 | function _recurseObject(obj, path) { 305 | for (var k in obj) { // eslint-disable-line guard-for-in 306 | var newPath = clone(path) 307 | newPath.push(k) 308 | if (typeof obj[k] === 'object' && obj[k] !== null) { 309 | _recurseObject(obj[k], newPath) 310 | } else { 311 | if (!obj.hasOwnProperty(k)) { 312 | continue 313 | } 314 | cb(obj[k], k, newPath) 315 | } 316 | } 317 | } 318 | } 319 | -------------------------------------------------------------------------------- /src/postAggregation.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var _ = require('./lodash') 4 | 5 | var aggregation = require('./aggregation') 6 | 7 | module.exports = function (/* service */) { 8 | return { 9 | post: post, 10 | sortByKey: sortByKey, 11 | limit: limit, 12 | squash: squash, 13 | change: change, 14 | changeMap: changeMap, 15 | } 16 | 17 | function post(query, parent, cb) { 18 | query.data = cloneIfLocked(parent) 19 | return Promise.resolve(cb(query, parent)) 20 | } 21 | 22 | function sortByKey(query, parent, desc) { 23 | query.data = cloneIfLocked(parent) 24 | query.data = _.sortBy(query.data, function (d) { 25 | return d.key 26 | }) 27 | if (desc) { 28 | query.data.reverse() 29 | } 30 | } 31 | 32 | // Limit results to n, or from start to end 33 | function limit(query, parent, start, end) { 34 | query.data = cloneIfLocked(parent) 35 | if (_.isUndefined(end)) { 36 | end = start || 0 37 | start = 0 38 | } else { 39 | start = start || 0 40 | end = end || query.data.length 41 | } 42 | query.data = query.data.splice(start, end - start) 43 | } 44 | 45 | // Squash results to n, or from start to end 46 | function squash(query, parent, start, end, aggObj, label) { 47 | query.data = cloneIfLocked(parent) 48 | start = start || 0 49 | end = end || query.data.length 50 | var toSquash = query.data.splice(start, end - start) 51 | var squashed = { 52 | key: label || 'Other', 53 | value: {}, 54 | } 55 | _.recurseObject(aggObj, function (val, key, path) { 56 | var items = [] 57 | _.forEach(toSquash, function (record) { 58 | items.push(_.get(record.value, path)) 59 | }) 60 | _.set(squashed.value, path, aggregation.aggregators[val](items)) 61 | }) 62 | query.data.splice(start, 0, squashed) 63 | } 64 | 65 | function change(query, parent, start, end, aggObj) { 66 | query.data = cloneIfLocked(parent) 67 | start = start || 0 68 | end = end || query.data.length 69 | var obj = { 70 | key: [query.data[start].key, query.data[end].key], 71 | value: {}, 72 | } 73 | _.recurseObject(aggObj, function (val, key, path) { 74 | var changePath = _.clone(path) 75 | changePath.pop() 76 | changePath.push(key + 'Change') 77 | _.set(obj.value, changePath, _.get(query.data[end].value, path) - _.get(query.data[start].value, path)) 78 | }) 79 | query.data = obj 80 | } 81 | 82 | function changeMap(query, parent, aggObj, defaultNull) { 83 | defaultNull = _.isUndefined(defaultNull) ? 0 : defaultNull 84 | query.data = cloneIfLocked(parent) 85 | _.recurseObject(aggObj, function (val, key, path) { 86 | var changePath = _.clone(path) 87 | var fromStartPath = _.clone(path) 88 | var fromEndPath = _.clone(path) 89 | 90 | changePath.pop() 91 | fromStartPath.pop() 92 | fromEndPath.pop() 93 | 94 | changePath.push(key + 'Change') 95 | fromStartPath.push(key + 'ChangeFromStart') 96 | fromEndPath.push(key + 'ChangeFromEnd') 97 | 98 | var start = _.get(query.data[0].value, path, defaultNull) 99 | var end = _.get(query.data[query.data.length - 1].value, path, defaultNull) 100 | 101 | _.forEach(query.data, function (record, i) { 102 | var previous = query.data[i - 1] || query.data[0] 103 | _.set(query.data[i].value, changePath, _.get(record.value, path, defaultNull) - (previous ? _.get(previous.value, path, defaultNull) : defaultNull)) 104 | _.set(query.data[i].value, fromStartPath, _.get(record.value, path, defaultNull) - start) 105 | _.set(query.data[i].value, fromEndPath, _.get(record.value, path, defaultNull) - end) 106 | }) 107 | }) 108 | } 109 | } 110 | 111 | function cloneIfLocked(parent) { 112 | return parent.locked ? _.clone(parent.data) : parent.data 113 | } 114 | -------------------------------------------------------------------------------- /src/query.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var _ = require('./lodash') 4 | 5 | module.exports = function (service) { 6 | var reductiofy = require('./reductiofy')(service) 7 | var filters = require('./filters')(service) 8 | var postAggregation = require('./postAggregation')(service) 9 | 10 | var postAggregationMethods = _.keys(postAggregation) 11 | 12 | return function doQuery(queryObj) { 13 | var queryHash = JSON.stringify(queryObj) 14 | 15 | // Attempt to reuse an exact copy of this query that is present elsewhere 16 | for (var i = 0; i < service.columns.length; i++) { 17 | for (var j = 0; j < service.columns[i].queries.length; j++) { 18 | if (service.columns[i].queries[j].hash === queryHash) { 19 | return new Promise(function (resolve, reject) { // eslint-disable-line no-loop-func 20 | try { 21 | resolve(service.columns[i].queries[j]) 22 | } catch (err) { 23 | reject(err) 24 | } 25 | }) 26 | } 27 | } 28 | } 29 | 30 | var query = { 31 | // Original query passed in to query method 32 | original: queryObj, 33 | hash: queryHash, 34 | } 35 | 36 | // Default queryObj 37 | if (_.isUndefined(query.original)) { 38 | query.original = {} 39 | } 40 | // Default select 41 | if (_.isUndefined(query.original.select)) { 42 | query.original.select = { 43 | $count: true, 44 | } 45 | } 46 | // Default to groupAll 47 | query.original.groupBy = query.original.groupBy || true 48 | 49 | // Attach the query api to the query object 50 | query = newQueryObj(query) 51 | 52 | return createColumn(query) 53 | .then(makeCrossfilterGroup) 54 | .then(buildRequiredColumns) 55 | .then(setupDataListeners) 56 | .then(applyQuery) 57 | 58 | function createColumn(query) { 59 | // Ensure column is created 60 | return service.column({ 61 | key: query.original.groupBy, 62 | type: _.isUndefined(query.type) ? null : query.type, 63 | array: Boolean(query.array), 64 | }) 65 | .then(function () { 66 | // Attach the column to the query 67 | var column = service.column.find(query.original.groupBy) 68 | query.column = column 69 | column.queries.push(query) 70 | column.removeListeners.push(function () { 71 | return query.clear() 72 | }) 73 | return query 74 | }) 75 | } 76 | 77 | function makeCrossfilterGroup(query) { 78 | // Create the grouping on the columns dimension 79 | // Using Promise Resolve allows support for crossfilter async 80 | // TODO check if query already exists, and use the same base query // if possible 81 | return Promise.resolve(query.column.dimension.group()) 82 | .then(function (g) { 83 | query.group = g 84 | return query 85 | }) 86 | } 87 | 88 | function buildRequiredColumns(query) { 89 | var requiredColumns = filters.scanForDynamicFilters(query.original) 90 | // We need to scan the group for any filters that would require 91 | // the group to be rebuilt when data is added or removed in any way. 92 | if (requiredColumns.length) { 93 | return Promise.all(_.map(requiredColumns, function (columnKey) { 94 | return service.column({ 95 | key: columnKey, 96 | dynamicReference: query.group, 97 | }) 98 | })) 99 | .then(function () { 100 | return query 101 | }) 102 | } 103 | return query 104 | } 105 | 106 | function setupDataListeners(query) { 107 | // Here, we create a listener to recreate and apply the reducer to 108 | // the group anytime underlying data changes 109 | var stopDataListen = service.onDataChange(function () { 110 | return applyQuery(query) 111 | }) 112 | query.removeListeners.push(stopDataListen) 113 | 114 | // This is a similar listener for filtering which will (if needed) 115 | // run any post aggregations on the data after each filter action 116 | var stopFilterListen = service.onFilter(function () { 117 | return postAggregate(query) 118 | }) 119 | query.removeListeners.push(stopFilterListen) 120 | 121 | return query 122 | } 123 | 124 | function applyQuery(query) { 125 | return buildReducer(query) 126 | .then(applyReducer) 127 | .then(attachData) 128 | .then(postAggregate) 129 | } 130 | 131 | function buildReducer(query) { 132 | return reductiofy(query.original) 133 | .then(function (reducer) { 134 | query.reducer = reducer 135 | return query 136 | }) 137 | } 138 | 139 | function applyReducer(query) { 140 | return Promise.resolve(query.reducer(query.group)) 141 | .then(function () { 142 | return query 143 | }) 144 | } 145 | 146 | function attachData(query) { 147 | return Promise.resolve(query.group.all()) 148 | .then(function (data) { 149 | query.data = data 150 | return query 151 | }) 152 | } 153 | 154 | function postAggregate(query) { 155 | if (query.postAggregations.length > 1) { 156 | // If the query is used by 2+ post aggregations, we need to lock 157 | // it against getting mutated by the post-aggregations 158 | query.locked = true 159 | } 160 | return Promise.all(_.map(query.postAggregations, function (post) { 161 | return post() 162 | })) 163 | .then(function () { 164 | return query 165 | }) 166 | } 167 | 168 | function newQueryObj(q, parent) { 169 | var locked = false 170 | if (!parent) { 171 | parent = q 172 | q = {} 173 | locked = true 174 | } 175 | 176 | // Assign the regular query properties 177 | _.assign(q, { 178 | // The Universe for continuous promise chaining 179 | universe: service, 180 | // Crossfilter instance 181 | crossfilter: service.cf, 182 | 183 | // parent Information 184 | parent: parent, 185 | column: parent.column, 186 | dimension: parent.dimension, 187 | group: parent.group, 188 | reducer: parent.reducer, 189 | original: parent.original, 190 | hash: parent.hash, 191 | 192 | // It's own removeListeners 193 | removeListeners: [], 194 | 195 | // It's own postAggregations 196 | postAggregations: [], 197 | 198 | // Data method 199 | locked: locked, 200 | lock: lock, 201 | unlock: unlock, 202 | // Disposal method 203 | clear: clearQuery, 204 | }) 205 | 206 | _.forEach(postAggregationMethods, function (method) { 207 | q[method] = postAggregateMethodWrap(postAggregation[method]) 208 | }) 209 | 210 | return q 211 | 212 | function lock(set) { 213 | if (!_.isUndefined(set)) { 214 | q.locked = Boolean(set) 215 | return 216 | } 217 | q.locked = true 218 | } 219 | 220 | function unlock() { 221 | q.locked = false 222 | } 223 | 224 | function clearQuery() { 225 | _.forEach(q.removeListeners, function (l) { 226 | l() 227 | }) 228 | return new Promise(function (resolve, reject) { 229 | try { 230 | resolve(q.group.dispose()) 231 | } catch (err) { 232 | reject(err) 233 | } 234 | }) 235 | .then(function () { 236 | q.column.queries.splice(q.column.queries.indexOf(q), 1) 237 | // Automatically recycle the column if there are no queries active on it 238 | if (!q.column.queries.length) { 239 | return service.clear(q.column.key) 240 | } 241 | }) 242 | .then(function () { 243 | return service 244 | }) 245 | } 246 | 247 | function postAggregateMethodWrap(postMethod) { 248 | return function () { 249 | var args = Array.prototype.slice.call(arguments) 250 | var sub = {} 251 | newQueryObj(sub, q) 252 | args.unshift(sub, q) 253 | 254 | q.postAggregations.push(function () { 255 | Promise.resolve(postMethod.apply(null, args)) 256 | .then(postAggregateChildren) 257 | }) 258 | 259 | return Promise.resolve(postMethod.apply(null, args)) 260 | .then(postAggregateChildren) 261 | 262 | function postAggregateChildren() { 263 | return postAggregate(sub) 264 | .then(function () { 265 | return sub 266 | }) 267 | } 268 | } 269 | } 270 | } 271 | } 272 | } 273 | -------------------------------------------------------------------------------- /src/reductioAggregators.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | // var _ = require('./lodash') // _ is defined but never used 4 | 5 | module.exports = { 6 | shorthandLabels: { 7 | $count: 'count', 8 | $sum: 'sum', 9 | $avg: 'avg', 10 | $min: 'min', 11 | $max: 'max', 12 | $med: 'med', 13 | $sumSq: 'sumSq', 14 | $std: 'std', 15 | }, 16 | aggregators: { 17 | $count: $count, 18 | $sum: $sum, 19 | $avg: $avg, 20 | $min: $min, 21 | $max: $max, 22 | $med: $med, 23 | $sumSq: $sumSq, 24 | $std: $std, 25 | $valueList: $valueList, 26 | $dataList: $dataList, 27 | }, 28 | } 29 | 30 | // Aggregators 31 | 32 | function $count(reducer/* , value */) { 33 | return reducer.count(true) 34 | } 35 | 36 | function $sum(reducer, value) { 37 | return reducer.sum(value) 38 | } 39 | 40 | function $avg(reducer, value) { 41 | return reducer.avg(value) 42 | } 43 | 44 | function $min(reducer, value) { 45 | return reducer.min(value) 46 | } 47 | 48 | function $max(reducer, value) { 49 | return reducer.max(value) 50 | } 51 | 52 | function $med(reducer, value) { 53 | return reducer.median(value) 54 | } 55 | 56 | function $sumSq(reducer, value) { 57 | return reducer.sumOfSq(value) 58 | } 59 | 60 | function $std(reducer, value) { 61 | return reducer.std(value) 62 | } 63 | 64 | function $valueList(reducer, value) { 65 | return reducer.valueList(value) 66 | } 67 | 68 | function $dataList(reducer/* , value */) { 69 | return reducer.dataList(true) 70 | } 71 | 72 | // TODO histograms 73 | // TODO exceptions 74 | -------------------------------------------------------------------------------- /src/reductiofy.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var reductio = require('reductio') 4 | 5 | var _ = require('./lodash') 6 | var rAggregators = require('./reductioAggregators') 7 | // var expressions = require('./expressions') // exporession is defined but never used 8 | var aggregation = require('./aggregation') 9 | 10 | module.exports = function (service) { 11 | var filters = require('./filters')(service) 12 | 13 | return function reductiofy(query) { 14 | var reducer = reductio() 15 | // var groupBy = query.groupBy // groupBy is defined but never used 16 | aggregateOrNest(reducer, query.select) 17 | 18 | if (query.filter) { 19 | var filterFunction = filters.makeFunction(query.filter) 20 | if (filterFunction) { 21 | reducer.filter(filterFunction) 22 | } 23 | } 24 | 25 | return Promise.resolve(reducer) 26 | 27 | // This function recursively find the first level of reductio methods in 28 | // each object and adds that reduction method to reductio 29 | function aggregateOrNest(reducer, selects) { 30 | // Sort so nested values are calculated last by reductio's .value method 31 | var sortedSelectKeyValue = _.sortBy( 32 | _.map(selects, function (val, key) { 33 | return { 34 | key: key, 35 | value: val, 36 | } 37 | }), 38 | function (s) { 39 | if (rAggregators.aggregators[s.key]) { 40 | return 0 41 | } 42 | return 1 43 | }) 44 | 45 | // dive into each key/value 46 | return _.forEach(sortedSelectKeyValue, function (s) { 47 | // Found a Reductio Aggregation 48 | if (rAggregators.aggregators[s.key]) { 49 | // Build the valueAccessorFunction 50 | var accessor = aggregation.makeValueAccessor(s.value) 51 | // Add the reducer with the ValueAccessorFunction to the reducer 52 | reducer = rAggregators.aggregators[s.key](reducer, accessor) 53 | return 54 | } 55 | 56 | // Found a top level key value that is not an aggregation or a 57 | // nested object. This is unacceptable. 58 | if (!_.isObject(s.value)) { 59 | console.error('Nested selects must be an object', s.key) 60 | return 61 | } 62 | 63 | // It's another nested object, so just repeat this process on it 64 | aggregateOrNest(reducer.value(s.key), s.value) 65 | }) 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/universe.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | var _ = require('./lodash') 4 | 5 | module.exports = universe 6 | 7 | function universe(data, options) { 8 | var service = { 9 | options: _.assign({}, options), 10 | columns: [], 11 | filters: {}, 12 | dataListeners: [], 13 | filterListeners: [], 14 | } 15 | 16 | var cf = require('./crossfilter')(service) 17 | var filters = require('./filters')(service) 18 | 19 | data = cf.generateColumns(data) 20 | 21 | return cf.build(data) 22 | .then(function (data) { 23 | service.cf = data 24 | return _.assign(service, { 25 | add: cf.add, 26 | remove: cf.remove, 27 | column: require('./column')(service), 28 | query: require('./query')(service), 29 | filter: filters.filter, 30 | filterAll: filters.filterAll, 31 | applyFilters: filters.applyFilters, 32 | clear: require('./clear')(service), 33 | destroy: require('./destroy')(service), 34 | onDataChange: onDataChange, 35 | onFilter: onFilter, 36 | }) 37 | }) 38 | 39 | function onDataChange(cb) { 40 | service.dataListeners.push(cb) 41 | return function () { 42 | service.dataListeners.splice(service.dataListeners.indexOf(cb), 1) 43 | } 44 | } 45 | 46 | function onFilter(cb) { 47 | service.filterListeners.push(cb) 48 | return function () { 49 | service.filterListeners.splice(service.filterListeners.indexOf(cb), 1) 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /test/clear.spec.js: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import universe from '../src/universe' 4 | import data from './fixtures/data' 5 | 6 | test('can clear all columns', async t => { 7 | const u = await universe(data) 8 | 9 | await u.column(['type', 'total']) 10 | t.is(u.columns.length, 2) 11 | 12 | await u.clear() 13 | 14 | t.deepEqual(u.columns, []) 15 | }) 16 | 17 | test('can remove a single column', async t => { 18 | const u = await universe(data) 19 | 20 | await u.column('type') 21 | 22 | t.is(u.columns.length, 1) 23 | await u.clear('type') 24 | 25 | t.is(u.columns.length, 0) 26 | }) 27 | 28 | test('can remove a single column based on multiple keys', async t => { 29 | const u = await universe(data) 30 | 31 | await u.column({ 32 | key: ['type', 'total', 'quantity', 'tip'] 33 | }) 34 | t.is(u.columns.length, 1) 35 | 36 | await u.clear({ 37 | key: ['type', 'total', 'quantity', 'tip'] 38 | }) 39 | t.is(u.columns.length, 0) 40 | }) 41 | 42 | test('can remove multiple columns', async t => { 43 | const u = await universe(data) 44 | 45 | await u.column(['type', 'total']) 46 | t.is(u.columns.length, 2) 47 | 48 | await u.clear(['type', 'total']) 49 | t.is(u.columns.length, 0) 50 | }) 51 | -------------------------------------------------------------------------------- /test/column.spec.js: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import universe from '../src/universe' 4 | import data from './fixtures/data' 5 | 6 | test('has the columns properties', async t => { 7 | const u = await universe(data) 8 | t.deepEqual(u.columns, []) 9 | }) 10 | 11 | test('has the column method', async t => { 12 | const u = await universe(data) 13 | t.is(typeof u.column, 'function') 14 | }) 15 | 16 | test('can add a column without a default type of string', async t => { 17 | const u = await universe(data) 18 | const res = await u.column('type') 19 | t.is(res.columns[0].key, 'type') 20 | t.is(res.columns[0].type, 'string') 21 | t.is(typeof res.columns[0].dimension, 'object') 22 | }) 23 | 24 | test('can add a column with a specified type', async t => { 25 | const u = await universe(data) 26 | const res = await u.column({ 27 | key: 'productIDs', 28 | array: true 29 | }) 30 | 31 | t.is(res.columns[0].key, 'productIDs') 32 | t.is(res.columns[0].type, 'array') 33 | t.is(typeof res.columns[0].dimension, 'object') 34 | }) 35 | 36 | test('can add a column with a complex key', async t => { 37 | const u = await universe(data) 38 | 39 | const res = await u.column({ 40 | key: ['type', 'total', 'quantity', 'tip'] 41 | }) 42 | 43 | t.deepEqual(res.columns[0].key, ['type', 'total', 'quantity', 'tip']) 44 | t.is(res.columns[0].type, 'complex') 45 | t.is(typeof res.columns[0].dimension, 'object') 46 | }) 47 | 48 | test('can add a column with nested string format', async t => { 49 | const u = await universe(data) 50 | 51 | const keyString = `productIDs[0]` 52 | 53 | const res = await u.column(keyString) 54 | 55 | t.deepEqual(res.columns[0].key, keyString) 56 | t.is(res.columns[0].type, 'complex') 57 | t.is(typeof res.columns[0].dimension, 'object') 58 | }) 59 | 60 | test('can add a column with a callback function', async t => { 61 | const u = await universe(data) 62 | 63 | const keyFn = d => { 64 | return `${d.type} - ${d.total}` 65 | } 66 | 67 | const res = await u.column(keyFn) 68 | 69 | t.deepEqual(res.columns[0].key, keyFn) 70 | t.is(res.columns[0].type, 'complex') 71 | t.is(typeof res.columns[0].dimension, 'object') 72 | }) 73 | 74 | test('can try to create the same column multiple times, but still only create one', async t => { 75 | const u = await universe(data) 76 | 77 | await Promise.all([ 78 | u.column({ 79 | key: ['type', 'total'] 80 | }), 81 | u.column({ 82 | key: ['type', 'total'] 83 | }), 84 | u.column({ 85 | key: ['type', 'total'] 86 | }), 87 | u.column({ 88 | key: ['type', 'total'] 89 | }), 90 | u.column({ 91 | key: ['type', 'total'] 92 | }), 93 | u.column({ 94 | key: ['type', 'total'] 95 | }), 96 | u.column({ 97 | key: ['type', 'total'] 98 | }), 99 | u.column({ 100 | key: ['type', 'total'] 101 | }), 102 | u.column({ 103 | key: ['type', 'total'] 104 | }), 105 | u.column({ 106 | key: ['type', 'total'] 107 | }), 108 | u.column({ 109 | key: ['type', 'total'] 110 | }), 111 | u.column({ 112 | key: ['type', 'total'] 113 | }) 114 | ]) 115 | 116 | t.is(u.columns.length, 1) 117 | }) 118 | -------------------------------------------------------------------------------- /test/destroy.spec.js: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import universe from '../src/universe' 4 | import data from './fixtures/data' 5 | 6 | test('can destroy the universe a few times over', async () => { 7 | const u = await universe(data) 8 | 9 | await u.query({ 10 | groupBy: 'type', 11 | select: { 12 | $count: true 13 | } 14 | }) 15 | 16 | await u.query({ 17 | groupBy: 'total', 18 | select: { 19 | $count: true 20 | } 21 | }) 22 | 23 | await u.query({ 24 | groupBy: 'tip', 25 | select: { 26 | $count: true 27 | } 28 | }) 29 | 30 | await u.destroy() 31 | 32 | await u.add(data) 33 | 34 | await u.query({ 35 | groupBy: 'type', 36 | select: { 37 | $count: true 38 | } 39 | }) 40 | 41 | await u.query({ 42 | groupBy: 'total', 43 | select: { 44 | $count: true 45 | } 46 | }) 47 | 48 | await u.query({ 49 | groupBy: 'tip', 50 | select: { 51 | $count: true 52 | } 53 | }) 54 | 55 | await u.destroy() 56 | 57 | await u.add(data) 58 | 59 | await u.query({ 60 | groupBy: 'type', 61 | select: { 62 | $count: true 63 | } 64 | }) 65 | 66 | await u.query({ 67 | groupBy: 'total', 68 | select: { 69 | $count: true 70 | } 71 | }) 72 | 73 | await u.query({ 74 | groupBy: 'tip', 75 | select: { 76 | $count: true 77 | } 78 | }) 79 | 80 | await u.destroy() 81 | }) 82 | -------------------------------------------------------------------------------- /test/filter-all.spec.js: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import universe from '../src/universe' 4 | import data from './fixtures/data' 5 | 6 | test('has the filterAll method', async t => { 7 | const u = await universe(data) 8 | t.is(typeof u.filterAll, 'function') 9 | }) 10 | 11 | test('can filterAll', async t => { 12 | const u = await universe(data) 13 | 14 | const q = await u.query({ 15 | groupBy: 'tip', 16 | select: { 17 | $count: true 18 | } 19 | }) 20 | 21 | t.deepEqual(q.data, [ 22 | {key: 0, value: {count: 8}}, 23 | {key: 100, value: {count: 3}}, 24 | {key: 200, value: {count: 1}} 25 | ]) 26 | 27 | await u.filter('type', 'cash') 28 | 29 | t.deepEqual(q.data, [ 30 | {key: 0, value: {count: 2}}, 31 | {key: 100, value: {count: 0}}, 32 | {key: 200, value: {count: 0}} 33 | ]) 34 | 35 | t.is(u.filters.type.value, 'cash') 36 | 37 | await u.filterAll() 38 | 39 | t.deepEqual(u.filters, {}) 40 | 41 | t.deepEqual(q.data, [ 42 | {key: 0, value: {count: 8}}, 43 | {key: 100, value: {count: 3}}, 44 | {key: 200, value: {count: 1}} 45 | ]) 46 | }) 47 | -------------------------------------------------------------------------------- /test/filter.spec.js: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import universe from '../src/universe' 4 | import data from './fixtures/data' 5 | 6 | test('has the filter method', async t => { 7 | const u = await universe(data) 8 | t.is(typeof u.filter, 'function') 9 | }) 10 | 11 | test('can filter', async t => { 12 | const u = await universe(data) 13 | 14 | const q = await u.query({ 15 | groupBy: 'type', 16 | select: { 17 | $count: 'true', 18 | $sum: 'total' 19 | }, 20 | filter: { 21 | $or: [{ 22 | total: { 23 | $gt: 50 24 | } 25 | }, { 26 | quantity: { 27 | $gt: 1 28 | } 29 | }] 30 | } 31 | }) 32 | 33 | t.deepEqual(q.data, [ 34 | {key: 'cash', value: {count: 2, sum: 300}}, 35 | {key: 'tab', value: {count: 8, sum: 920}}, 36 | {key: 'visa', value: {count: 2, sum: 500}} 37 | ]) 38 | }) 39 | 40 | test('can not filter on a non-existent column', async t => { 41 | const u = await universe(data) 42 | 43 | await u.query({ 44 | groupBy: 'total', 45 | select: { 46 | $max: 'total' 47 | } 48 | }) 49 | 50 | try { 51 | await u.filter('someOtherColumn', { 52 | $gt: 95 53 | }) 54 | } catch (err) { 55 | t.is(String(err), 'Error: Column key does not exist in data!') 56 | } 57 | }) 58 | 59 | test('can filter based on a single column that is not defined yet, then recycle that column', async t => { 60 | const u = await universe(data) 61 | 62 | const q = await u.query({ 63 | groupBy: 'tip', 64 | select: { 65 | $max: 'total' 66 | } 67 | }) 68 | 69 | await u.filter('total', { 70 | $gt: 95 71 | }) 72 | 73 | t.deepEqual(q.data, [ 74 | {key: 0, value: {max: 200, valueList: [100, 200]}}, 75 | {key: 100, value: {max: 200, valueList: [190, 190, 200]}}, 76 | {key: 200, value: {max: 300, valueList: [300]}} 77 | ]) 78 | 79 | await u.filter('total') 80 | t.is(u.columns.length, 1) 81 | }) 82 | 83 | test('can filter based using filterFunction', async t => { 84 | const u = await universe(data) 85 | 86 | const q = await u.query({ 87 | groupBy: 'tip', 88 | select: { 89 | $max: 'total' 90 | } 91 | }) 92 | 93 | await u.filter('total', d => d > 95) 94 | 95 | t.deepEqual(q.data, [ 96 | {key: 0, value: {max: 200, valueList: [100, 200]}}, 97 | {key: 100, value: {max: 200, valueList: [190, 190, 200]}}, 98 | {key: 200, value: {max: 300, valueList: [300]}} 99 | ]) 100 | }) 101 | 102 | test('can filter based using filterFunction, together with exact', async t => { 103 | const u = await universe(data) 104 | 105 | const q = await u.query({ 106 | groupBy: 'tip', 107 | select: { 108 | $max: 'total' 109 | } 110 | }) 111 | 112 | await u.filter('type', 'visa', true, true) 113 | t.is(typeof u.filters.type.value, 'string') 114 | 115 | await u.filter('total', d => d > 95) 116 | t.is(typeof u.filters.total.value, 'function') 117 | t.is(typeof u.filters.type.value, 'string') 118 | 119 | // t.deepEqual(q.data[0], {key: 0, value: {max: null, valueList: []}}) 120 | t.deepEqual(q.data[1], {key: 100, value: {max: 200, valueList: [200]}}) 121 | t.deepEqual(q.data[2], {key: 200, value: {max: 300, valueList: [300]}}) 122 | }) 123 | 124 | // see https://github.com/crossfilter/universe/issues/20 125 | test('can filter based using filterFunction, works in reverse', async t => { 126 | const u = await universe(data) 127 | 128 | const q = await u.query({ 129 | groupBy: 'tip', 130 | select: { 131 | $max: 'total' 132 | } 133 | }) 134 | 135 | await u.filter('total', d => d > 95) 136 | t.is(typeof u.filters.total.value, 'function') 137 | 138 | await u.filter('type', 'visa', true, true) 139 | t.is(typeof u.filters.total.value, 'function') 140 | t.is(typeof u.filters.type.value, 'string') 141 | 142 | // t.deepEqual(q.data[0], {key: 0, value: {max: null, valueList: []}}) 143 | t.deepEqual(q.data[1], {key: 100, value: {max: 200, valueList: [200]}}) 144 | t.deepEqual(q.data[2], {key: 200, value: {max: 300, valueList: [300]}}) 145 | }) 146 | 147 | test('can filter based on a complex column regardless of key order', async t => { 148 | const u = await universe(data) 149 | 150 | const q = await u.query({ 151 | groupBy: ['tip', 'total'], 152 | select: { 153 | $max: 'total' 154 | } 155 | }) 156 | 157 | await u.filter(['total', 'tip'], { 158 | $gt: 95 159 | }) 160 | 161 | t.deepEqual(q.data, [ 162 | {key: [0, 100], value: {valueList: [100], max: 100}}, 163 | {key: [0, 200], value: {valueList: [200], max: 200}}, 164 | {key: [0, 90], value: {valueList: [90, 90, 90, 90, 90, 90], max: 90}}, 165 | {key: [100, 190], value: {valueList: [190, 190], max: 190}}, 166 | {key: [100, 200], value: {valueList: [200], max: 200}}, 167 | {key: [200, 300], value: {valueList: [300], max: 300}} 168 | ]) 169 | }) 170 | 171 | test('can filter using $column data', async t => { 172 | const u = await universe(data) 173 | 174 | const q = await u.query({ 175 | groupBy: 'tip', 176 | filter: { 177 | type: { 178 | $last: { 179 | $column: 'type' 180 | } 181 | } 182 | } 183 | }) 184 | 185 | t.deepEqual(q.data, [ 186 | {key: 0, value: {count: 8}}, 187 | {key: 100, value: {count: 3}}, 188 | {key: 200, value: {count: 1}} 189 | ]) 190 | }) 191 | 192 | test('can filter using all $data', async t => { 193 | const u = await universe(data) 194 | 195 | const q = await u.query({ 196 | groupBy: 'type', 197 | select: { 198 | $count: 'true', 199 | }, 200 | filter: { 201 | date: { 202 | $gt: { 203 | '$get(date)': { 204 | '$nthPct(50)': '$data' 205 | } 206 | } 207 | } 208 | } 209 | }) 210 | 211 | t.deepEqual(q.data, [ 212 | {key: 'cash', value: {count: 1}}, 213 | {key: 'tab', value: {count: 3}}, 214 | {key: 'visa', value: {count: 1}} 215 | ]) 216 | }) 217 | 218 | test('can not remove colum that is used in dynamic filter', async t => { 219 | const u = await universe(data) 220 | 221 | await u.query({ 222 | groupBy: 'type', 223 | select: { 224 | $count: 'true', 225 | }, 226 | filter: { 227 | date: { 228 | $gt: { 229 | '$get(date)': { 230 | '$nth(2)': { 231 | $column: 'date' 232 | } 233 | } 234 | } 235 | } 236 | } 237 | }) 238 | 239 | await u.clear('date') 240 | t.is(u.columns.length, 2) 241 | }) 242 | 243 | test('can toggle filters using simple values', async t => { 244 | const u = await universe(data) 245 | 246 | const q = await u.query({ 247 | groupBy: 'tip', 248 | select: { 249 | $count: true 250 | } 251 | }) 252 | 253 | await u.filter('type', 'cash') 254 | t.is(u.filters.type.value, 'cash') 255 | t.deepEqual(q.data, [ 256 | {key: 0, value: {count: 2}}, 257 | {key: 100, value: {count: 0}}, 258 | {key: 200, value: {count: 0}} 259 | ]) 260 | 261 | await u.filter('type', 'visa') 262 | t.deepEqual(u.filters.type.value, ['visa', 'cash']) 263 | t.deepEqual(q.data, [ 264 | {key: 0, value: {count: 2}}, 265 | {key: 100, value: {count: 1}}, 266 | {key: 200, value: {count: 1}} 267 | ]) 268 | 269 | await u.filter('type', 'tab') 270 | t.deepEqual(u.filters.type.value, ['tab', 'visa', 'cash']) 271 | t.deepEqual(q.data, [ 272 | {key: 0, value: {count: 8}}, 273 | {key: 100, value: {count: 3}}, 274 | {key: 200, value: {count: 1}} 275 | ]) 276 | 277 | await u.filter('type', 'visa') 278 | t.deepEqual(u.filters.type.value, ['tab', 'cash']) 279 | t.deepEqual(q.data, [ 280 | {key: 0, value: {count: 8}}, 281 | {key: 100, value: {count: 2}}, 282 | {key: 200, value: {count: 0}} 283 | ]) 284 | }) 285 | 286 | // see https://github.com/crossfilter/universe/issues/20 287 | test('can toggle multiple filters using simple values', async t => { 288 | const u = await universe(data) 289 | 290 | await u.query({ 291 | groupBy: 'tip', 292 | select: { 293 | $count: true 294 | } 295 | }) 296 | 297 | await u.filter('type', 'cash') 298 | t.is(u.filters.type.value, 'cash') 299 | 300 | await u.filter('type', 'visa') 301 | t.deepEqual(u.filters.type.value, ['visa', 'cash']) 302 | 303 | await u.filter('quantity', 2) 304 | t.deepEqual(u.filters.quantity.value, 2) 305 | t.deepEqual(u.filters.type.value, ['visa', 'cash']) 306 | }) 307 | 308 | test('can toggle filters using an array as a range', async t => { 309 | const u = await universe(data) 310 | 311 | const q = await u.query({ 312 | groupBy: 'type', 313 | select: { 314 | $count: true 315 | } 316 | }) 317 | 318 | await u.filter('total', [85, 101], true) 319 | t.deepEqual(q.data, [ 320 | {key: 'cash', value: {count: 1}}, 321 | {key: 'tab', value: {count: 6}}, 322 | {key: 'visa', value: {count: 0}} 323 | ]) 324 | 325 | await u.filter('total', [85, 91], true) 326 | t.deepEqual(q.data, [ 327 | {key: 'cash', value: {count: 0}}, 328 | {key: 'tab', value: {count: 6}}, 329 | {key: 'visa', value: {count: 0}} 330 | ]) 331 | }) 332 | 333 | test('can toggle filters using an array as an include', async t => { 334 | const u = await universe(data) 335 | 336 | const q = await u.query({ 337 | groupBy: 'type', 338 | select: { 339 | $count: true 340 | } 341 | }) 342 | 343 | await u.filter('total', [90, 100]) 344 | 345 | t.deepEqual(q.data, [ 346 | {key: 'cash', value: {count: 1}}, 347 | {key: 'tab', value: {count: 6}}, 348 | {key: 'visa', value: {count: 0}} 349 | ]) 350 | 351 | await u.filter('total', [90, 300, 200]) 352 | 353 | t.deepEqual(q.data, [ 354 | {key: 'cash', value: {count: 2}}, 355 | {key: 'tab', value: {count: 0}}, 356 | {key: 'visa', value: {count: 2}} 357 | ]) 358 | }) 359 | 360 | test('can forcefully replace filters', async t => { 361 | const u = await universe(data) 362 | 363 | await u.query({ 364 | groupBy: 'tip', 365 | select: { 366 | $count: true 367 | } 368 | }) 369 | 370 | await u.filter('type', 'cash') 371 | t.is(u.filters.type.value, 'cash') 372 | 373 | await u.filter('type', ['tab', 'visa'], false, true) 374 | t.deepEqual(u.filters.type.value, ['tab', 'visa']) 375 | }) 376 | 377 | test('can apply many filters in one go', async t => { 378 | const u = await universe(data) 379 | 380 | await u.query({ 381 | groupBy: 'tip', 382 | select: { 383 | $count: true 384 | } 385 | }) 386 | 387 | await u.filter('type', 'cash') 388 | t.is(u.filters.type.value, 'cash') 389 | 390 | await u.filterAll([{ 391 | column: 'type', 392 | value: 'visa', 393 | }, { 394 | column: 'quantity', 395 | value: [200, 500], 396 | isRange: true, 397 | }]) 398 | 399 | t.deepEqual(u.filters, { 400 | type: { 401 | value: ['visa', 'cash'], 402 | replace: undefined, 403 | type: 'inclusive' 404 | }, 405 | quantity: { 406 | value: [200, 500], 407 | replace: true, 408 | type: 'range' 409 | } 410 | } 411 | )}) 412 | -------------------------------------------------------------------------------- /test/fixtures/data.js: -------------------------------------------------------------------------------- 1 | module.exports = [{ 2 | date: '2011-11-14T16:17:54Z', 3 | quantity: 2, 4 | total: 190, 5 | tip: 100, 6 | type: 'tab', 7 | productIDs: ['001'] 8 | }, { 9 | date: '2011-11-14T16:20:19Z', 10 | quantity: 2, 11 | total: 190, 12 | tip: 100, 13 | type: 'tab', 14 | productIDs: ['001', '005'] 15 | }, { 16 | date: '2011-11-14T16:28:54Z', 17 | quantity: 1, 18 | total: 300, 19 | tip: 200, 20 | type: 'visa', 21 | productIDs: ['004', '005'] 22 | }, { 23 | date: '2011-11-14T16:30:43Z', 24 | quantity: 2, 25 | total: 90, 26 | tip: 0, 27 | type: 'tab', 28 | productIDs: ['001', '002'] 29 | }, { 30 | date: '2011-11-14T16:48:46Z', 31 | quantity: 2, 32 | total: 90, 33 | tip: 0, 34 | type: 'tab', 35 | productIDs: ['005'] 36 | }, { 37 | date: '2011-11-14T16:53:41Z', 38 | quantity: 2, 39 | total: 90, 40 | tip: 0, 41 | type: 'tab', 42 | productIDs: ['001', '004', '005'] 43 | }, { 44 | date: '2011-11-14T16:54:06Z', 45 | quantity: 1, 46 | total: 100, 47 | tip: 0, 48 | type: 'cash', 49 | productIDs: ['001', '002', '003', '004', '005'] 50 | }, { 51 | date: '2011-11-14T16:58:03Z', 52 | quantity: 2, 53 | total: 90, 54 | tip: 0, 55 | type: 'tab', 56 | productIDs: ['001'] 57 | }, { 58 | date: '2011-11-14T17:07:21Z', 59 | quantity: 2, 60 | total: 90, 61 | tip: 0, 62 | type: 'tab', 63 | productIDs: ['004', '005'] 64 | }, { 65 | date: '2011-11-14T17:22:59Z', 66 | quantity: 2, 67 | total: 90, 68 | tip: 0, 69 | type: 'tab', 70 | productIDs: ['001', '002', '004', '005'] 71 | }, { 72 | date: '2011-11-14T17:25:45Z', 73 | quantity: 2, 74 | total: 200, 75 | tip: 0, 76 | type: 'cash', 77 | productIDs: ['002'] 78 | }, { 79 | date: '2011-11-14T17:29:52Z', 80 | quantity: 1, 81 | total: 200, 82 | tip: 100, 83 | type: 'visa', 84 | productIDs: ['004'] 85 | }] 86 | -------------------------------------------------------------------------------- /test/post-aggregation.spec.js: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import universe from '../src/universe' 4 | import data from './fixtures/data' 5 | 6 | test('can do chained general post aggregations', async t => { 7 | const u = await universe(data) 8 | 9 | const before = await u.query({ 10 | groupBy: 'type' 11 | }) 12 | 13 | before.lock() 14 | t.deepEqual(before.data, [ 15 | {key: 'cash', value: {count: 2}}, 16 | {key: 'tab', value: {count: 8}}, 17 | {key: 'visa', value: {count: 2}} 18 | ]) 19 | 20 | const after = await before.post(q => { 21 | q.data[0].value.count += 10 22 | q.data[2].key += '_test' 23 | }) 24 | 25 | after.lock() 26 | t.deepEqual(after.data, [ 27 | {key: 'cash', value: {count: 12}}, 28 | {key: 'tab', value: {count: 8}}, 29 | {key: 'visa_test', value: {count: 2}} 30 | ]) 31 | 32 | const after2 = await after.post(q => { 33 | q.data[0].value.count += 10 34 | q.data[2].key += '_test' 35 | }) 36 | after2.lock() 37 | 38 | await u.filter('total', '100') 39 | 40 | t.deepEqual(before.data, [ 41 | {key: 'cash', value: {count: 1}}, 42 | {key: 'tab', value: {count: 0}}, 43 | {key: 'visa', value: {count: 0}} 44 | ]) 45 | t.deepEqual(after.data, [ 46 | {key: 'cash', value: {count: 11}}, 47 | {key: 'tab', value: {count: 0}}, 48 | {key: 'visa_test', value: {count: 0}} 49 | ]) 50 | t.deepEqual(after2.data, [ 51 | {key: 'cash', value: {count: 21}}, 52 | {key: 'tab', value: {count: 0}}, 53 | {key: 'visa_test_test', value: {count: 0}} 54 | ]) 55 | }) 56 | 57 | test('works after filtering', async t => { 58 | const u = await universe(data) 59 | 60 | const q = await u.query({ 61 | groupBy: 'total', 62 | }) 63 | 64 | const res = await q.changeMap({ 65 | count: true 66 | }) 67 | 68 | t.deepEqual(res.data[0].value.countChangeFromEnd, 5) 69 | await u.filter('type', 'cash') 70 | 71 | t.deepEqual(res.data[0].value.countChangeFromEnd, 0) 72 | }) 73 | 74 | test('can sortByKey ascending and descending', async t => { 75 | const u = await universe(data) 76 | 77 | const q = await u.query({ 78 | groupBy: 'type' 79 | }) 80 | 81 | const res = await q.sortByKey(true) 82 | t.deepEqual(res.data[0].key, 'visa') 83 | 84 | await res.sortByKey() 85 | t.deepEqual(res.data[0].key, 'cash') 86 | 87 | await res.sortByKey(true) 88 | await u.filter('total', 100) 89 | 90 | t.deepEqual(res.data[0].key, 'visa') 91 | }) 92 | 93 | test('can limit', async t => { 94 | const u = await universe(data) 95 | 96 | const q = await u.query({ 97 | groupBy: 'total', 98 | }) 99 | 100 | const res = await q.limit(2, null) 101 | 102 | t.deepEqual(res.data[0].key, 190) 103 | }) 104 | 105 | test('can squash', async t => { 106 | const u = await universe(data) 107 | 108 | const q = await u.query({ 109 | groupBy: 'total', 110 | select: { 111 | $sum: 'total' 112 | } 113 | }) 114 | 115 | const res = await q.squash(2, 4, { 116 | sum: '$sum' 117 | }, 'SQUASHED!!!') 118 | 119 | t.deepEqual(res.data[2].key, 'SQUASHED!!!') 120 | t.deepEqual(res.data[2].value.sum, 780) 121 | }) 122 | 123 | test('can find change based on index for multiple values', async t => { 124 | const u = await universe(data) 125 | 126 | const q = await u.query({ 127 | groupBy: 'total', 128 | select: { 129 | $count: true, 130 | $sum: 'total', 131 | } 132 | }) 133 | 134 | const res = await q.change(2, 4, { 135 | count: true, 136 | sum: true 137 | }) 138 | 139 | t.deepEqual(res.data, { 140 | key: [190, 300], 141 | value: { 142 | countChange: -1, 143 | sumChange: -80, 144 | } 145 | }) 146 | }) 147 | 148 | test('can create a changeMap', async t => { 149 | const u = await universe(data) 150 | 151 | const q = await u.query({ 152 | groupBy: 'total', 153 | select: { 154 | $count: true, 155 | $sum: 'total', 156 | } 157 | }) 158 | 159 | const res = await q.changeMap({ 160 | count: true, 161 | sum: true, 162 | }) 163 | 164 | t.deepEqual(res.data, [ 165 | {key: 90, 166 | value: { 167 | count: 6, 168 | sum: 540, 169 | countChange: 0, 170 | countChangeFromStart: 0, 171 | countChangeFromEnd: 5, 172 | sumChange: 0, 173 | sumChangeFromStart: 0, 174 | sumChangeFromEnd: 240 175 | } 176 | }, 177 | {key: 100, 178 | value: { 179 | count: 1, 180 | sum: 100, 181 | countChange: -5, 182 | countChangeFromStart: -5, 183 | countChangeFromEnd: 0, 184 | sumChange: -440, 185 | sumChangeFromStart: -440, 186 | sumChangeFromEnd: -200 187 | } 188 | }, 189 | {key: 190, 190 | value: { 191 | count: 2, 192 | sum: 380, 193 | countChange: 1, 194 | countChangeFromStart: -4, 195 | countChangeFromEnd: 1, 196 | sumChange: 280, 197 | sumChangeFromStart: -160, 198 | sumChangeFromEnd: 80 199 | } 200 | }, 201 | {key: 200, 202 | value: { 203 | count: 2, 204 | sum: 400, 205 | countChange: 0, 206 | countChangeFromStart: -4, 207 | countChangeFromEnd: 1, 208 | sumChange: 20, 209 | sumChangeFromStart: -140, 210 | sumChangeFromEnd: 100 211 | } 212 | }, 213 | {key: 300, 214 | value: { 215 | count: 1, 216 | sum: 300, 217 | countChange: -1, 218 | countChangeFromStart: -5, 219 | countChangeFromEnd: 0, 220 | sumChange: -100, 221 | sumChangeFromStart: -240, 222 | sumChangeFromEnd: 0 223 | } 224 | } 225 | ]) 226 | }) 227 | -------------------------------------------------------------------------------- /test/query.dynamicData.spec.js: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import universe from '../src/universe' 4 | import data from './fixtures/data' 5 | 6 | test('can add data to an existing query', async t => { 7 | const u = await universe(data) 8 | 9 | const q = await u.query({ 10 | groupBy: 'type', 11 | select: { 12 | $count: 'true', 13 | $sum: 'total' 14 | }, 15 | }) 16 | 17 | t.deepEqual(q.data, [ 18 | {key: 'cash', value: {count: 2, sum: 300}}, 19 | {key: 'tab', value: {count: 8, sum: 920}}, 20 | {key: 'visa', value: {count: 2, sum: 500}} 21 | ]) 22 | 23 | await u.add([{ 24 | date: '2012-11-14T17:29:52Z', 25 | quantity: 100, 26 | total: 50000, 27 | tip: 999, 28 | type: 'visa', 29 | productIDs: ['004'] 30 | }, { 31 | date: '2012-11-14T17:29:52Z', 32 | quantity: 100, 33 | total: 400, 34 | tip: 600, 35 | type: 'other', 36 | productIDs: ['004'] 37 | }]) 38 | 39 | t.deepEqual(q.data, [ 40 | {key: 'cash', value: {count: 2, sum: 300}}, 41 | {key: 'other', value: {count: 1, sum: 400}}, 42 | {key: 'tab', value: {count: 8, sum: 920}}, 43 | {key: 'visa', value: {count: 3, sum: 50500}}, 44 | ]) 45 | }) 46 | 47 | test('can add new data to dynamic filters', async t => { 48 | const u = await universe(data) 49 | 50 | const q = await u.query({ 51 | groupBy: 'type', 52 | select: { 53 | $count: 'true', 54 | $sum: 'total' 55 | }, 56 | filter: { 57 | date: { 58 | $eq: { 59 | $last: { 60 | $column: 'date' 61 | } 62 | } 63 | } 64 | } 65 | }) 66 | 67 | t.deepEqual(q.data, [ 68 | {key: 'cash', value: {count: 0, sum: 0}}, 69 | {key: 'tab', value: {count: 0, sum: 0}}, 70 | {key: 'visa', value: {count: 1, sum: 200}} 71 | ]) 72 | 73 | await u.add([{ 74 | date: '2012-11-14T17:29:52Z', 75 | quantity: 100, 76 | total: 50000, 77 | tip: 999, 78 | type: 'visa', 79 | productIDs: ['004'] 80 | }]) 81 | 82 | t.deepEqual(q.data, [ 83 | {key: 'cash', value: {count: 0, sum: 0}}, 84 | {key: 'tab', value: {count: 0, sum: 0}}, 85 | {key: 'visa', value: {count: 1, sum: 50000}}, 86 | ]) 87 | }) 88 | 89 | test('can query using the valueList aggregation', async t => { 90 | const u = await universe(data) 91 | 92 | const q = await u.query({ 93 | groupBy: 'type', 94 | select: { 95 | $valueList: 'total', 96 | } 97 | }) 98 | 99 | await u.add([{ 100 | date: '2012-11-14T17:29:52Z', 101 | quantity: 100, 102 | total: 50000, 103 | tip: 999, 104 | type: 'visa', 105 | productIDs: ['004'] 106 | }]) 107 | 108 | t.deepEqual(q.data, [ 109 | {key: 'cash', value: {valueList: [100, 200]}}, 110 | {key: 'tab', value: {valueList: [90, 90, 90, 90, 90, 90, 190, 190]}}, 111 | {key: 'visa', value: {valueList: [200, 300, 50000]}} 112 | ]) 113 | }) 114 | -------------------------------------------------------------------------------- /test/query.spec.js: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import universe from '../src/universe' 4 | import data from './fixtures/data' 5 | 6 | test('has the query method', async t => { 7 | const u = await universe(data) 8 | t.deepEqual(typeof u.query, 'function') 9 | }) 10 | 11 | test('can create ad-hoc dimensions for each column', async () => { 12 | const u = await universe(data) 13 | 14 | await u.query({ 15 | groupBy: 'date', 16 | select: {} 17 | }) 18 | 19 | await u.query({ 20 | groupBy: 'quantity', 21 | select: {} 22 | }) 23 | 24 | await u.query({ 25 | groupBy: 'total', 26 | select: {} 27 | }) 28 | 29 | await u.query({ 30 | groupBy: 'tip', 31 | select: {} 32 | }) 33 | 34 | await u.query({ 35 | groupBy: 'type', 36 | select: {} 37 | }) 38 | 39 | await u.query({ 40 | groupBy: 'productIDs', 41 | select: {} 42 | }) 43 | 44 | await u.query({ 45 | groupBy: ['productIDs', 'date'], 46 | select: {} 47 | }) 48 | }) 49 | 50 | test('Defaults to counting each record', async t => { 51 | const u = await universe(data) 52 | 53 | const q = await u.query() 54 | 55 | t.deepEqual(q.data, [ 56 | {key: 0, value: {count: 1}}, 57 | {key: 1, value: {count: 1}}, 58 | {key: 2, value: {count: 1}}, 59 | {key: 3, value: {count: 1}}, 60 | {key: 4, value: {count: 1}}, 61 | {key: 5, value: {count: 1}}, 62 | {key: 6, value: {count: 1}}, 63 | {key: 7, value: {count: 1}}, 64 | {key: 8, value: {count: 1}}, 65 | {key: 9, value: {count: 1}}, 66 | {key: 10, value: {count: 1}}, 67 | {key: 11, value: {count: 1}} 68 | ]) 69 | }) 70 | 71 | test('supports all reductio aggregations', async t => { 72 | const u = await universe(data) 73 | 74 | const q = await u.query({ 75 | select: { 76 | $count: true, 77 | $sum: 'total', 78 | $avg: 'total', 79 | $min: 'total', 80 | $max: 'total', 81 | $med: 'total', 82 | $sumSq: 'total', 83 | $std: 'total', 84 | } 85 | }) 86 | 87 | t.deepEqual(q.data, [ 88 | {key: 0, value: {count: 1, sum: 190, avg: 190, valueList: [190], median: 190, min: 190, max: 190, sumOfSq: 36100, std: 0}}, 89 | {key: 1, value: {count: 1, sum: 190, avg: 190, valueList: [190], median: 190, min: 190, max: 190, sumOfSq: 36100, std: 0}}, 90 | {key: 2, value: {count: 1, sum: 300, avg: 300, valueList: [300], median: 300, min: 300, max: 300, sumOfSq: 90000, std: 0}}, 91 | {key: 3, value: {count: 1, sum: 90, avg: 90, valueList: [90], median: 90, min: 90, max: 90, sumOfSq: 8100, std: 0}}, 92 | {key: 4, value: {count: 1, sum: 90, avg: 90, valueList: [90], median: 90, min: 90, max: 90, sumOfSq: 8100, std: 0}}, 93 | {key: 5, value: {count: 1, sum: 90, avg: 90, valueList: [90], median: 90, min: 90, max: 90, sumOfSq: 8100, std: 0}}, 94 | {key: 6, value: {count: 1, sum: 100, avg: 100, valueList: [100], median: 100, min: 100, max: 100, sumOfSq: 10000, std: 0}}, 95 | {key: 7, value: {count: 1, sum: 90, avg: 90, valueList: [90], median: 90, min: 90, max: 90, sumOfSq: 8100, std: 0}}, 96 | {key: 8, value: {count: 1, sum: 90, avg: 90, valueList: [90], median: 90, min: 90, max: 90, sumOfSq: 8100, std: 0}}, 97 | {key: 9, value: {count: 1, sum: 90, avg: 90, valueList: [90], median: 90, min: 90, max: 90, sumOfSq: 8100, std: 0}}, 98 | {key: 10, value: {count: 1, sum: 200, avg: 200, valueList: [200], median: 200, min: 200, max: 200, sumOfSq: 40000, std: 0}}, 99 | {key: 11, value: {count: 1, sum: 200, avg: 200, valueList: [200], median: 200, min: 200, max: 200, sumOfSq: 40000, std: 0} 100 | } 101 | ]) 102 | }) 103 | 104 | test('supports column aggregations with arrays', async t => { 105 | const u = await universe(data) 106 | 107 | const q = await u.query({ 108 | select: { 109 | $sum: { 110 | $sum: ['tip', 'total'] 111 | }, 112 | } 113 | }) 114 | 115 | t.deepEqual(q.data, [ 116 | {key: 0, value: {sum: 290}}, 117 | {key: 1, value: {sum: 290}}, 118 | {key: 2, value: {sum: 500}}, 119 | {key: 3, value: {sum: 90}}, 120 | {key: 4, value: {sum: 90}}, 121 | {key: 5, value: {sum: 90}}, 122 | {key: 6, value: {sum: 100}}, 123 | {key: 7, value: {sum: 90}}, 124 | {key: 8, value: {sum: 90}}, 125 | {key: 9, value: {sum: 90}}, 126 | {key: 10, value: {sum: 200}}, 127 | {key: 11, value: {sum: 300}} 128 | ]) 129 | }) 130 | 131 | test('supports column aggregations with objects', async t => { 132 | const u = await universe(data) 133 | 134 | const q = await u.query({ 135 | select: { 136 | $sum: { 137 | $sum: { 138 | $max: ['tip', 'total'], 139 | $min: ['quantity', 'total'] 140 | } 141 | }, 142 | } 143 | }) 144 | 145 | t.deepEqual(q.data, [ 146 | {key: 0, value: {sum: 192}}, 147 | {key: 1, value: {sum: 192}}, 148 | {key: 2, value: {sum: 301}}, 149 | {key: 3, value: {sum: 92}}, 150 | {key: 4, value: {sum: 92}}, 151 | {key: 5, value: {sum: 92}}, 152 | {key: 6, value: {sum: 101}}, 153 | {key: 7, value: {sum: 92}}, 154 | {key: 8, value: {sum: 92}}, 155 | {key: 9, value: {sum: 92}}, 156 | {key: 10, value: {sum: 202}}, 157 | {key: 11, value: {sum: 201}} 158 | ]) 159 | }) 160 | 161 | test('supports column aggregations using string syntax', async t => { 162 | const u = await universe(data) 163 | 164 | const q = await u.query({ 165 | select: { 166 | $sum: '$sum($max(tip,total), $min(quantity,total))' 167 | } 168 | }) 169 | 170 | t.deepEqual(q.data, [ 171 | {key: 0, value: {sum: 192}}, 172 | {key: 1, value: {sum: 192}}, 173 | {key: 2, value: {sum: 301}}, 174 | {key: 3, value: {sum: 92}}, 175 | {key: 4, value: {sum: 92}}, 176 | {key: 5, value: {sum: 92}}, 177 | {key: 6, value: {sum: 101}}, 178 | {key: 7, value: {sum: 92}}, 179 | {key: 8, value: {sum: 92}}, 180 | {key: 9, value: {sum: 92}}, 181 | {key: 10, value: {sum: 202}}, 182 | {key: 11, value: {sum: 201}} 183 | ]) 184 | }) 185 | 186 | test('supports groupBy', async t => { 187 | const u = await universe(data) 188 | 189 | const q = await u.query({ 190 | groupBy: 'type' 191 | }) 192 | 193 | t.deepEqual(q.data, [ 194 | {key: 'cash', value: {count: 2}}, 195 | {key: 'tab', value: {count: 8}}, 196 | {key: 'visa', value: {count: 2}} 197 | ]) 198 | }) 199 | 200 | test('can query using the valueList aggregation', async t => { 201 | const u = await universe(data) 202 | 203 | const q = await u.query({ 204 | groupBy: 'type', 205 | select: { 206 | $valueList: 'total', 207 | } 208 | }) 209 | 210 | t.deepEqual(q.data, [ 211 | {key: 'cash', value: {valueList: [100, 200]}}, 212 | {key: 'tab', value: {valueList: [90, 90, 90, 90, 90, 90, 190, 190]}}, 213 | {key: 'visa', value: {valueList: [200, 300]}}]) 214 | }) 215 | 216 | test('can query using the dataList aggregation', async t => { 217 | const u = await universe(data) 218 | 219 | const q = await u.query({ 220 | groupBy: 'type', 221 | select: { 222 | $dataList: true, 223 | } 224 | }) 225 | 226 | t.deepEqual(q.data, [{ 227 | key: 'cash', 228 | value: { 229 | dataList: [ 230 | {date: '2011-11-14T16:54:06Z', quantity: 1, total: 100, tip: 0, type: 'cash', productIDs: ['001', '002', '003', '004', '005']}, 231 | {date: '2011-11-14T17:25:45Z', quantity: 2, total: 200, tip: 0, type: 'cash', productIDs: ['002']} 232 | ] 233 | } 234 | }, { 235 | key: 'tab', 236 | value: { 237 | dataList: [ 238 | {date: '2011-11-14T16:17:54Z', quantity: 2, total: 190, tip: 100, type: 'tab', productIDs: ['001']}, 239 | {date: '2011-11-14T16:20:19Z', quantity: 2, total: 190, tip: 100, type: 'tab', productIDs: ['001', '005']}, 240 | {date: '2011-11-14T16:30:43Z', quantity: 2, total: 90, tip: 0, type: 'tab', productIDs: ['001', '002']}, 241 | {date: '2011-11-14T16:48:46Z', quantity: 2, total: 90, tip: 0, type: 'tab', productIDs: ['005']}, 242 | {date: '2011-11-14T16:53:41Z', quantity: 2, total: 90, tip: 0, type: 'tab', productIDs: ['001', '004', '005']}, 243 | {date: '2011-11-14T16:58:03Z', quantity: 2, total: 90, tip: 0, type: 'tab', productIDs: ['001']}, 244 | {date: '2011-11-14T17:07:21Z', quantity: 2, total: 90, tip: 0, type: 'tab', productIDs: ['004', '005']}, 245 | {date: '2011-11-14T17:22:59Z', quantity: 2, total: 90, tip: 0, type: 'tab', productIDs: ['001', '002', '004', '005']} 246 | ] 247 | } 248 | }, { 249 | key: 'visa', 250 | value: { 251 | dataList: [ 252 | {date: '2011-11-14T16:28:54Z', quantity: 1, total: 300, tip: 200, type: 'visa', productIDs: ['004', '005']}, 253 | {date: '2011-11-14T17:29:52Z', quantity: 1, total: 200, tip: 100, type: 'visa', productIDs: ['004']} 254 | ]} 255 | }]) 256 | }) 257 | 258 | // TODO: This isn't completely possible yet, reductio will need to support aliases for all aggregations first. As of this commit, it is only available on `count` 259 | // test('supports nested aliases', function(){ 260 | // return universe(data).then(function(u){ 261 | // return u.query({ 262 | // groupBy: 'type', 263 | // select: { 264 | // my: { 265 | // awesome: { 266 | // column: { 267 | // $count: true 268 | // } 269 | // } 270 | // } 271 | // }, 272 | // }) 273 | // }) 274 | // .then(function(res){ 275 | // console.log(res) 276 | // t.deepEqual(res.data, [ 277 | // {key: "cash", value: {count: 2}}, 278 | // {key: "tab", value: {count: 8}}, 279 | // {key: "visa", value: {count: 2}} 280 | // ]) 281 | // }) 282 | // }) 283 | 284 | test('supports multi aggregation', async t => { 285 | const u = await universe(data) 286 | 287 | const q = await u.query({ 288 | select: { 289 | tip: {$sum: 'tip'}, 290 | total: {$sum: 'total'} 291 | } 292 | }) 293 | 294 | t.deepEqual(q.data, [ 295 | {key: 0, value: {tip: {sum: 100}, total: {sum: 190}}}, 296 | {key: 1, value: {tip: {sum: 100}, total: {sum: 190}}}, 297 | {key: 2, value: {tip: {sum: 200}, total: {sum: 300}}}, 298 | {key: 3, value: {tip: {sum: 0}, total: {sum: 90}}}, 299 | {key: 4, value: {tip: {sum: 0}, total: {sum: 90}}}, 300 | {key: 5, value: {tip: {sum: 0}, total: {sum: 90}}}, 301 | {key: 6, value: {tip: {sum: 0}, total: {sum: 100}}}, 302 | {key: 7, value: {tip: {sum: 0}, total: {sum: 90}}}, 303 | {key: 8, value: {tip: {sum: 0}, total: {sum: 90}}}, 304 | {key: 9, value: {tip: {sum: 0}, total: {sum: 90}}}, 305 | {key: 10, value: {tip: {sum: 0}, total: {sum: 200}}}, 306 | {key: 11, value: {tip: {sum: 100}, total: {sum: 200}}} 307 | ]) 308 | }) 309 | 310 | test('can dispose of a query manually', async t => { 311 | const u = await universe(data) 312 | 313 | const q = await u.query({ 314 | groupBy: 'type', 315 | select: { 316 | $count: true 317 | } 318 | }) 319 | 320 | t.deepEqual(q.universe.columns.length, 1) 321 | await q.clear() 322 | 323 | t.deepEqual(u.columns.length, 0) 324 | }) 325 | -------------------------------------------------------------------------------- /test/universe.spec.js: -------------------------------------------------------------------------------- 1 | import test from 'ava' 2 | 3 | import crossfilter from 'crossfilter2' 4 | 5 | import universe from '../src/universe' 6 | import data from './fixtures/data' 7 | 8 | test('is a function', async t => { 9 | t.is(typeof universe, 'function') 10 | }) 11 | 12 | test('requires a crossfilter instance', t => { 13 | return universe() 14 | .then(res => { 15 | return t.is(typeof res, 'object') 16 | }) 17 | .catch(err => { 18 | return t.is(typeof err, 'object') 19 | }) 20 | }) 21 | 22 | test('can accept a crossfilter instance', () => { 23 | return universe(crossfilter(data)) 24 | }) 25 | 26 | test('can accept an array of data points', () => { 27 | return universe(data) 28 | }) 29 | 30 | test('can create generated columns using an accessor function', async t => { 31 | const u = await universe(data, { 32 | generatedColumns: { 33 | totalAndTip: d => d.total + d.tip 34 | } 35 | }) 36 | 37 | const res = await u.query({ 38 | groupBy: 'totalAndTip' 39 | }) 40 | 41 | t.deepEqual(res.data, [ 42 | {key: 90, value: {count: 6}}, 43 | {key: 100, value: {count: 1}}, 44 | {key: 200, value: {count: 1}}, 45 | {key: 290, value: {count: 2}}, 46 | {key: 300, value: {count: 1}}, 47 | {key: 500, value: {count: 1}} 48 | ]) 49 | }) 50 | --------------------------------------------------------------------------------