├── .gitignore ├── LICENSE ├── README.md ├── dist ├── browser │ ├── 1.datamodel.js │ ├── 2.datamodel.js │ ├── 8b88c9bd5247bbf5828f.worker.js │ ├── c5e4ba14b90211ccd6b7.module.wasm │ └── datamodel.js ├── esm │ ├── web │ │ └── src │ │ │ ├── constants │ │ │ ├── aggregations.d.ts │ │ │ ├── aggregations.js │ │ │ ├── context.d.ts │ │ │ ├── context.js │ │ │ ├── errorMessage.d.ts │ │ │ ├── errorMessage.js │ │ │ ├── fields.d.ts │ │ │ ├── fields.js │ │ │ ├── filtering-modes.d.ts │ │ │ ├── filtering-modes.js │ │ │ ├── invalid-data.d.ts │ │ │ ├── invalid-data.js │ │ │ ├── miscellaneous.d.ts │ │ │ ├── miscellaneous.js │ │ │ ├── selections.d.ts │ │ │ ├── selections.js │ │ │ ├── sort.d.ts │ │ │ └── sort.js │ │ │ ├── contexts │ │ │ ├── _to_do_context_registry.d.ts │ │ │ ├── _to_do_context_registry.js │ │ │ └── wasm │ │ │ │ ├── fields │ │ │ │ ├── categorical.d.ts │ │ │ │ ├── categorical.js │ │ │ │ ├── continuous.d.ts │ │ │ │ ├── continuous.js │ │ │ │ ├── field.d.ts │ │ │ │ ├── field.js │ │ │ │ ├── row_id.d.ts │ │ │ │ ├── row_id.js │ │ │ │ ├── temporal.d.ts │ │ │ │ ├── temporal.js │ │ │ │ ├── utils.d.ts │ │ │ │ └── utils.js │ │ │ │ ├── index.d.ts │ │ │ │ ├── index.js │ │ │ │ └── utils │ │ │ │ ├── index.d.ts │ │ │ │ ├── index.js │ │ │ │ ├── select-utils.d.ts │ │ │ │ └── select-utils.js │ │ │ ├── contracts │ │ │ ├── context.d.ts │ │ │ ├── context.js │ │ │ ├── data.d.ts │ │ │ ├── data.js │ │ │ ├── dataWranglers.d.ts │ │ │ ├── dataWranglers.js │ │ │ ├── datamodel.d.ts │ │ │ ├── datamodel.js │ │ │ ├── field.d.ts │ │ │ ├── field.js │ │ │ ├── query.d.ts │ │ │ ├── query.js │ │ │ ├── tasker.d.ts │ │ │ └── tasker.js │ │ │ ├── data │ │ │ ├── dataParsers │ │ │ │ ├── auto-resolver.d.ts │ │ │ │ ├── auto-resolver.js │ │ │ │ ├── dsv-arr.d.ts │ │ │ │ ├── dsv-arr.js │ │ │ │ ├── dsv-str.d.ts │ │ │ │ ├── dsv-str.js │ │ │ │ ├── flat-json.d.ts │ │ │ │ ├── flat-json.js │ │ │ │ ├── store.d.ts │ │ │ │ └── store.js │ │ │ ├── defaultConverters │ │ │ │ ├── autoConverter.d.ts │ │ │ │ ├── autoConverter.js │ │ │ │ ├── dsvArrayConverter.d.ts │ │ │ │ ├── dsvArrayConverter.js │ │ │ │ ├── dsvStringConverter.d.ts │ │ │ │ ├── dsvStringConverter.js │ │ │ │ ├── index.d.ts │ │ │ │ ├── index.js │ │ │ │ ├── jsonConverter.d.ts │ │ │ │ ├── jsonConverter.js │ │ │ │ ├── store.d.ts │ │ │ │ └── store.js │ │ │ ├── fieldParsers │ │ │ │ ├── categorical.d.ts │ │ │ │ ├── categorical.js │ │ │ │ ├── continuous.d.ts │ │ │ │ ├── continuous.js │ │ │ │ ├── field-contract.d.ts │ │ │ │ ├── field-contract.js │ │ │ │ ├── index.d.ts │ │ │ │ ├── index.js │ │ │ │ ├── temporal.d.ts │ │ │ │ └── temporal.js │ │ │ ├── index.d.ts │ │ │ ├── index.js │ │ │ ├── invalid │ │ │ │ ├── invalid.d.ts │ │ │ │ ├── invalid.js │ │ │ │ ├── store.d.ts │ │ │ │ └── store.js │ │ │ └── utils │ │ │ │ ├── column-major.d.ts │ │ │ │ ├── column-major.js │ │ │ │ ├── date-time-formatter.d.ts │ │ │ │ ├── date-time-formatter.js │ │ │ │ ├── helper.d.ts │ │ │ │ ├── helper.js │ │ │ │ ├── index.d.ts │ │ │ │ ├── index.js │ │ │ │ ├── transform-data.d.ts │ │ │ │ └── transform-data.js │ │ │ ├── export.d.ts │ │ │ ├── export.js │ │ │ ├── index.d.ts │ │ │ ├── index.js │ │ │ ├── main.d.ts │ │ │ ├── main.js │ │ │ ├── operations │ │ │ ├── derivations.d.ts │ │ │ ├── derivations.js │ │ │ ├── dispose.d.ts │ │ │ ├── dispose.js │ │ │ ├── propagation │ │ │ │ ├── index.d.ts │ │ │ │ └── index.js │ │ │ └── wranglers │ │ │ │ ├── index.d.ts │ │ │ │ └── index.js │ │ │ └── worker │ │ │ ├── create-pool.d.ts │ │ │ ├── create-pool.js │ │ │ ├── index.d.ts │ │ │ ├── index.js │ │ │ ├── pool-executor.d.ts │ │ │ ├── pool-executor.js │ │ │ ├── transform-data.worker.d.ts │ │ │ └── transform-data.worker.js │ └── webassembly │ │ └── pkg │ │ ├── datamodel_wasm.d.ts │ │ ├── datamodel_wasm.js │ │ └── datamodel_wasm_bg.wasm └── node │ ├── 1.datamodel.js │ ├── 2.datamodel.js │ ├── 2cf9e3dd7ff6959fdd40.worker.js │ ├── bff0e3323ebd9a29ab52.module.wasm │ └── datamodel.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | # PLATFORM 2 | # ======== 3 | # All exclusions that are specific to the NPM, GIT, IDE and Operating Systems. 4 | 5 | # Ignore scripts for automatic docs deployment 6 | am_*.sh 7 | 8 | # - Do not allow installed node modules to be committed. Doing `npm install -d` will bring them in root or other places. 9 | node_modules 10 | 11 | # - Do not commit any log file from anywhere 12 | *.log 13 | 14 | # - Prevent addition of OS specific file explorer files 15 | Thumbs.db 16 | .DS_Store 17 | 18 | 19 | # PROJECT 20 | # ======= 21 | # Configuration pertaining to project specific repository structure. 22 | 23 | # - Prevent Sublime text IDE files from being commited to repository 24 | *.sublime-* 25 | 26 | # - Prevent NetBeans IDE files from being commited to repository 27 | nbproject/ 28 | 29 | # - Allow sublime text project file to be commited in the development directory. 30 | !/develop/*.sublime-project 31 | 32 | # - We use file names with `attr-*` for chart attribute tutorial files. These files remain in the `xt-attrs` submodule, 33 | # but are copied to a temp folder for building docs along with the tutorials from `develop/docs`. They should not be 34 | # tracked in the main project 35 | develop/docs/attr-* 36 | 37 | # - Prevent extra `jshint` configuration from being committed from anywhere. Only `package.json` will be used as 38 | # accepted source of config. 39 | .jshintrc 40 | 41 | # - Prevent Modular build template files from being Added 42 | develop/template/modular 43 | 44 | # - Prevent diff backups from SourceTree from showing as commit. 45 | *.BACKUP.* 46 | *.BASE.* 47 | *.LOCAL.* 48 | *.REMOTE.* 49 | *.orig 50 | 51 | # Ignore .swp files in case vim is getting used 52 | *.swp 53 | 54 | # Ignore .tmp folder created for mocha-webpack test cases 55 | /.tmp/ 56 | 57 | # Ignore docs folder created by jsdoc 58 | /docs 59 | 60 | # Ignore coverage folder created by karma 61 | coverage 62 | 63 | jquery 64 | 65 | package-lock.json 66 | 67 | develop 68 | 69 | extras 70 | .vscode -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | End User License Agreement for Muze and DataModel (WebAssembly version) 2 | 3 | Licensor: Muze (Charts.com), InfoSoft Global (P) Ltd. 4 | Web : https://www.muzejs.org 5 | Email : muze [at] muzejs.org 6 | 7 | END-USER LICENSE AGREEMENT FOR THIS SOFTWARE 8 | 9 | This End-User License Agreement ("EULA") is a legal agreement between you ("Licensee"), either an individual or an 10 | entity, and the mentioned Licensor of this Software for the software product identified above, which includes computer 11 | software and may include associated media, printed materials, and "online" or electronic documentation ("Software"). 12 | By installing, copying, or otherwise using the Software, you agree to be bound by the terms of this EULA. If you do not 13 | agree to the terms of this EULA, do not install or use the Software. 14 | 15 | SOFTWARE LICENSE 16 | 17 | The Software is protected by copyright laws and international copyright treaties, as well as other intellectual property 18 | laws and treaties. The Software is licensed, not sold. The Software under this License is provided free of charge. 19 | Even though a license fee is not paid for the use of such software, it does not mean that there are no conditions for 20 | using such software. 21 | 22 | 1. GRANT OF LICENSE 23 | 24 | You are granted a non-exclusive License to Use the downloaded Software for any purposes 25 | for an unlimited period of time. 26 | 27 | Installation and Use: You may install and use an unlimited number of copies of the Software. 28 | 29 | Reproduction and Distribution: You may reproduce and distribute an unlimited number of copies of the Software either 30 | in whole or in part; each copy should include all copyright and trademark notices, and shall be accompanied by a 31 | copy of this EULA. Copies of the Software may be distributed as a standalone product or included with your own product. 32 | 33 | Commercial Use: You may use the Software for commercial purposes. 34 | 35 | Reverse engineering: You may not reverse engineer or disassemble the Software. 36 | 37 | 2. INTELLECTUAL PROPERTY RIGHTS 38 | 39 | This License does not transmit any intellectual rights on the Software. The Software and any copies that the Licensee is 40 | authorized by the Licensor to make are the intellectual property of and are owned by the Licensor. 41 | The Software is protected by copyright, including without limitation by Copyright Law and international treaty provisions. 42 | 43 | Any copies that the Licensee is permitted to make pursuant to this Agreement must contain the same copyright 44 | and other proprietary notices that appear on or in the Software. 45 | 46 | The structure, organization and code of the Software are the valuable trade secrets and confidential information 47 | of the Licensor. The Licensee agrees not to decompile, disassemble or otherwise attempt to discover the source code 48 | of the Software. 49 | 50 | Any attempts to reverse-engineer, copy, clone, modify or alter in any way the installer program without the 51 | Licensor’s specific approval are strictly prohibited. The Licensee is not authorized to use any plug-in or 52 | enhancement that permits to save modifications to a file with software licensed and distributed by the Licensor. 53 | 54 | Trademarks shall be used in accordance with accepted trademark practice, including identification 55 | of trademarks owners’ names. Trademarks can only be used to identify printed output produced by the Software 56 | and such use of any trademark does not give the Licensee any rights of ownership in that trademark. 57 | 58 | All title and copyrights in and to the Software (including but not limited to any images, 59 | photographs, animations, video, audio, music and text incorporated into the Software), the accompanying 60 | documentation, and any copies of the Software are owned by the Licensor of this Software. The Software is 61 | protected by copyright laws and international treaty provisions. Therefore, you must treat the Software like 62 | any other copyrighted material. 63 | 64 | LIMITED WARRANTY 65 | 66 | 1. NO WARRANTIES 67 | 68 | The Licensor of this Software expressly disclaims any warranty for the Software. 69 | The Software and any related documentation is provided "as is" without warranty of any kind, 70 | either express or implied, including, without limitation, the implied warranties or merchantability, 71 | fitness for a particular purpose, or non-infringement. The entire risk arising out of use or performance 72 | of the Software remains with you. 73 | 74 | 2. NO LIABILITY FOR DAMAGES 75 | 76 | In no event shall the Licensor of this Software be liable for any damages whatsoever 77 | (including, without limitation, damages for loss of business profits, business interruption, 78 | loss of business information, or any other pecuniary loss) arising out of the use of or inability to use 79 | this product, even if the Licensor of this Software has been advised of the possibility of such damages. 80 | Because some states/jurisdictions do not allow the exclusion or limitation of liability for consequential 81 | or incidental damages, the above limitation may not apply to you. 82 | 83 | [END OF LICENSE] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [](http://muzejs.org/muze-wa/eula) 2 | [](http://muzejs.org/muze-wa/eula) 3 | [](https://www.npmjs.com/package/@chartshq/datamodel) 4 | [](https://github.com/chartshq/datamodel/graphs/contributors) 5 | 6 | ## What is DataModel? 7 | 8 | DataModel is an in-browser representation of tabular data. It uses WebAssembly for high performance and works seamlessly with any JavaScript library. It supports [Relational Algebra](https://en.wikipedia.org/wiki/Relational_algebra) operators which enable you to run `select`, `group`, `sort` (and many more) operations on the data. 9 | 10 | The current version performs all the data operations like `filtering`, `aggregation`, etc. on **[WebAssembly](https://webassembly.org/)** which gives a **10x performance** boost compared to the [old JavaScript version](https://github.com/chartshq/datamodel-deprecated). 11 | 12 | It is written in [Rust Language](https://www.rust-lang.org/) to handle computation intensive data operations, which is then compiled to **[WebAssembly](https://webassembly.org/)**, thereby providing a native-like performance for data operations. 13 | 14 | DataModel can be used if you need an in-browser tabular data store for data analysis, visualization or just general use of data. 15 | 16 | ## Features 17 | 18 | * 🎉 Supports [**Relational Algebra**](https://en.wikipedia.org/wiki/Relational_algebra) operators e.g. `selection`, `projection`, `group`, `calculateVariable`, `sort` etc out-of-the-box. 19 | 20 | * 💎 Every operation creates **Immutable** DataModel instance and builds a Directed Acyclic Graph (DAG) which establishes auto interactivity. 21 | 22 | * 🚀 Uses **[WebAssembly](https://webassembly.org/)** for handling huge datasets and for **better performance**. 23 | 24 | * ⛺ Also works in **Nodejs** environment out-of-the-box. 25 | 26 | ## Installation 27 | 28 | ### CDN 29 | 30 | Insert the DataModel build into the `
`: 31 | 32 | ```html 33 | 34 | ``` 35 | 36 | ### NPM 37 | 38 | Install DataModel from NPM: 39 | 40 | ```bash 41 | $ npm install --save @chartshq/datamodel 42 | ``` 43 | 44 | As we're using [Worker](https://developer.mozilla.org/en-US/docs/Web/API/Worker) internally, so the [worker-loader](https://www.npmjs.com/package/worker-loader) needs to be installed as follows: 45 | 46 | ```bash 47 | $ npm install worker-loader --save-dev 48 | ``` 49 | 50 | And then within your webpack configuration object, you'll need to add the `worker-loader` to the list of module rules, like so: 51 | 52 | ```js 53 | module.exports = { 54 | module: { 55 | rules: [ 56 | // Add the following object to your module `rules` list. 57 | { 58 | test: /\.worker/, 59 | include: /datamodel/, 60 | loader: 'worker-loader', 61 | options: { 62 | inline: false, // If you want to make it inline, set to true. 63 | fallback: true 64 | }, 65 | }, 66 | ], 67 | } 68 | }; 69 | ``` 70 | 71 | You also can checkout our [datamodel-app-template](https://github.com/chartshq/datamodel-app-template) to try out the `DataModel` quickly through a boilerplate app. 72 | 73 | ## Getting Started 74 | 75 | Once the installation is done, please follow the steps below: 76 | 77 | 1. Prepare the data and the corresponding schema: 78 | 79 | ```js 80 | // Prepare the schema for data. 81 | const schema = [ 82 | { 83 | name: 'Name', 84 | type: 'dimension' 85 | }, 86 | { 87 | name: 'Maker', 88 | type: 'dimension' 89 | }, 90 | { 91 | name: 'Horsepower', 92 | type: 'measure', 93 | defAggFn: 'avg' 94 | }, 95 | { 96 | name: 'Origin', 97 | type: 'dimension' 98 | } 99 | ] 100 | 101 | // Prepare the data. 102 | const data = [ 103 | { 104 | "Name": "chevrolet chevelle malibu", 105 | "Maker": "chevrolet", 106 | "Horsepower": 130, 107 | "Origin": "USA" 108 | }, 109 | { 110 | "Name": "buick skylark 320", 111 | "Maker": "buick", 112 | "Horsepower": 165, 113 | "Origin": "USA" 114 | }, 115 | { 116 | "Name": "datsun pl510", 117 | "Maker": "datsun", 118 | "Horsepower": 88, 119 | "Origin": "Japan" 120 | } 121 | ] 122 | ``` 123 | 124 | 2. Import DataModel as follows: 125 | 126 | If you are using the npm package, import the package as below: 127 | ```js 128 | import Engine from '@chartshq/datamodel'; 129 | ``` 130 | 131 | If you are using it in NodeJS, then require it as below: 132 | ```js 133 | const Engine = require('@chartshq/datamodel').default; 134 | ``` 135 | 136 | If you are using CDN, then use it as follows: 137 | ```js 138 | const Engine = window.DataModel; 139 | ``` 140 | 141 | 3. Load the DataModel engine and pass the data and schema to `DataModel` constructor and create a new `DataModel` instance: 142 | 143 | ```js 144 | // As the DataModel are asynchronous, so we need to 145 | // use async-await syntax. 146 | async function myAsyncFn() { 147 | // Load the DataModel module. 148 | const DataModel = await Engine.onReady(); 149 | 150 | // Converts the raw data into a format 151 | // which DataModel can consume. 152 | const formattedData = await DataModel.loadData(data, schema); 153 | 154 | // Create a new DataModel instance with 155 | // the formatted data. 156 | const dm = new DataModel(formattedData); 157 | 158 | console.log(dm.getData().data); 159 | // Output: 160 | // [ 161 | // ["chevrolet chevelle malibu", "chevrolet", 130, "USA"], 162 | // ["buick skylark 320", "buick", 165, "USA"], 163 | // ["datsun pl510", "datsun", 88, "Japan"] 164 | // ] 165 | 166 | // Perform the selection operation. 167 | const selectDm = dm.select({ field: 'Origin', value: 'USA', operator: DataModel.ComparisonOperators.EQUAL }); 168 | console.log(selectDm.getData().data); 169 | // Output: 170 | // [ 171 | // ["chevrolet chevelle malibu", "chevrolet", 130, "USA], 172 | // ["buick skylark 320", "buick", 165, "USA] 173 | // ] 174 | 175 | // Perform the projection operation. 176 | const projectDm = dm.project(["Origin", "Maker"]); 177 | console.log(projectDm.getData().data); 178 | // Output: 179 | // [ 180 | // ["USA", "chevrolet"], 181 | // ["USA", "buick"], 182 | // ["Japan", "datsun"] 183 | // ] 184 | 185 | console.log(projectDm.getData().schema); 186 | // Output: 187 | // [ 188 | // {"name": "Origin","type": "dimension"}, 189 | // {"name": "Maker","type": "dimension"} 190 | // ] 191 | } 192 | 193 | myAsyncFn() 194 | .catch(console.error.bind(console)); 195 | 196 | ``` 197 | 198 | 4. Now dispose the DataModel instance if it's not needed: 199 | 200 | ```js 201 | // This also disposes all the datamodels which are created from it. 202 | dm.dispose(); 203 | ``` 204 | 205 | ## Documentation 206 | 207 | Find detailed documentation and API reference from [here](https://muzejs.org/docs/wa/latest/concepts/datamodel/introducing-datamodel). 208 | 209 | ## What has changed? 210 | 211 | DataModel 3.0.0 now has the core written in [Rust language](https://www.rust-lang.org/) and has been ported to **[WebAssembly](https://webassembly.org/)** bringing in a huge performance difference w.r.t to [previous version](https://github.com/chartshq/datamodel-deprecated), in terms of both data size and computing speed. While the JavaScript version is deprecated and no active development will take place there but critical bugs if raised would be taken and released in GitHub only. 212 | 213 | You can visit the JavaScript (deprecated) version here [https://github.com/chartshq/datamodel-deprecated](https://github.com/chartshq/datamodel-deprecated) 214 | 215 | ## Migrating from previous versions of DataModel 216 | 217 | Now the DataModel became asynchronous as opposed to being synchronous in the previous JavaScript version. 218 | 219 | ```js 220 | import Engine from '@chartshq/datamodel'; 221 | 222 | (async () => { 223 | // Load the DataModel module. 224 | const DataModel = await Engine.onReady(); 225 | 226 | // Converts the raw data into a format 227 | // which DataModel can consume. 228 | const formattedData = await DataModel.loadData(data, schema); 229 | 230 | // Create a new DataModel instance with 231 | // the formatted data. 232 | const dm = new DataModel(formattedData); 233 | })(); 234 | ``` 235 | 236 | ### **Changed APIs** 237 | 238 | - **select** 239 | 240 | DataModel deprecated version: 241 | 242 | ```js 243 | dm.select((fields) => { 244 | return fields.Origin.value === 'USA'; 245 | }); 246 | ``` 247 | 248 | Latest version: 249 | 250 | ```js 251 | dm.select({ 252 | field: 'Origin', 253 | operator: DataModel.ComparisonOperators.EQUAL, 254 | value: 'USA' 255 | }); 256 | ``` 257 | 258 | - **groupBy** 259 | 260 | DataModel deprecated version: 261 | 262 | ```js 263 | dm.groupBy(['Origin'], { 264 | Acceleration: 'avg' 265 | }); 266 | ``` 267 | 268 | Latest version: 269 | 270 | ```js 271 | dm.groupBy(['Origin'], [{ 272 | aggn: DataModel.AggregationFunctions.AVG, 273 | field: 'Acceleration' 274 | }]); 275 | ``` 276 | 277 | Supported data operations: 278 | 279 | - select 280 | - project 281 | - calculateVariable 282 | - sort 283 | - groupBy 284 | 285 | Upcoming data operations: 286 | 287 | - join 288 | - bin 289 | - compose 290 | - union 291 | - difference 292 | - ... many more ... 293 | 294 | For more details on APIs visit our [docs](https://muzejs.org/docs/wa/latest/concepts/datamodel/introducing-datamodel). 295 | 296 | ## License 297 | 298 | [Custom License](https://muzejs.org/muze-wa/eula) (Free to use) 299 | -------------------------------------------------------------------------------- /dist/browser/1.datamodel.js: -------------------------------------------------------------------------------- 1 | (window.webpackJsonpDataModel=window.webpackJsonpDataModel||[]).push([[1],[,,,,function(e,t,n){(function(e){var r=Object.getOwnPropertyDescriptors||function(e){for(var t=Object.keys(e),n={},r=0;r