├── src ├── string │ ├── index.ts │ └── stringUtils.ts ├── commonjs.ts ├── array │ ├── index.ts │ ├── utils.ts │ ├── joins.ts │ ├── transform.ts │ └── stats.ts ├── utils │ ├── index.ts │ ├── json-parser.ts │ ├── table.ts │ ├── dsv-parser.ts │ └── helpers.ts ├── index.ts ├── examples │ ├── esm │ │ ├── utils.mjs │ │ ├── array.mjs │ │ └── datapipe.mjs │ ├── cjs │ │ ├── utils.js │ │ ├── array.js │ │ └── datapipe.js │ └── test.js ├── _internals.ts ├── tests │ ├── table.spec.ts │ ├── data-pipe.spec.ts │ ├── string-utils.spec.ts │ ├── json-parser.spec.ts │ ├── utils-pipe.spec.ts │ ├── dsv-parser.spec.ts │ └── array.spec.ts ├── types.ts └── data-pipe.ts ├── setupJest.js ├── .eslintignore ├── typedoc.json ├── .prettierrc ├── .vscode ├── settings.json └── launch.json ├── copy-types.js ├── .npmignore ├── .eslintrc ├── rollup.config.dev.js ├── .gitignore ├── LICENSE ├── index.html ├── rollup.config.js ├── package.json ├── tsconfig.json ├── jest.config.js └── README.md /src/string/index.ts: -------------------------------------------------------------------------------- 1 | export * from './stringUtils'; 2 | -------------------------------------------------------------------------------- /setupJest.js: -------------------------------------------------------------------------------- 1 | global.fetch = require('jest-fetch-mock'); 2 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | **/*/*.d.ts 4 | ./*js 5 | -------------------------------------------------------------------------------- /typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "datapipe js doc", 3 | "inputFiles": ["./src/data-pipe.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /src/commonjs.ts: -------------------------------------------------------------------------------- 1 | export * from './index'; 2 | export * from './array'; 3 | export * from './string'; 4 | export * from './utils'; 5 | -------------------------------------------------------------------------------- /src/array/index.ts: -------------------------------------------------------------------------------- 1 | export * from './joins'; 2 | export * from './stats'; 3 | export * from './transform'; 4 | export * from './utils'; 5 | -------------------------------------------------------------------------------- /src/utils/index.ts: -------------------------------------------------------------------------------- 1 | export * from './dsv-parser'; 2 | export * from './json-parser'; 3 | export * from './helpers'; 4 | export * from './table'; 5 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 100, 3 | "trailingComma": "none", 4 | "arrowParens": "avoid", 5 | "parser": "typescript", 6 | "singleQuote": true, 7 | "tabWidth": 2 8 | } 9 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.tsdk": "node_modules\\typescript\\lib", 3 | "files.exclude": { 4 | "array": true, 5 | "docs": true, 6 | "string": true, 7 | "utils": true 8 | } 9 | } -------------------------------------------------------------------------------- /copy-types.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | 3 | ['array', 'string', 'utils'].forEach(subFolder => { 4 | fs.writeFileSync(`./${subFolder}/index.d.ts`, `export * from './${subFolder}'; 5 | export * from './types';`); 6 | fs.copyFileSync('./src/types.ts', `./${subFolder}/types.ts`) 7 | }); 8 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /src 2 | /node_modules 3 | .gitignore 4 | rollup.config.js 5 | jest.config.js 6 | *.tgz 7 | tsconfig.json 8 | ./copy-types.js 9 | 10 | # IDEs and editors 11 | /.idea 12 | .project 13 | .classpath 14 | .c9/ 15 | *.launch 16 | .settings/ 17 | *.sublime-workspace 18 | 19 | # IDE - VSCode 20 | .vscode/* 21 | 22 | /docs 23 | /md-docs 24 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "parser": "@typescript-eslint/parser", 4 | "plugins": [ 5 | "@typescript-eslint" 6 | ], 7 | "extends": [ 8 | "eslint:recommended", 9 | "plugin:@typescript-eslint/eslint-recommended", 10 | "plugin:@typescript-eslint/recommended" 11 | ], 12 | "rules": { 13 | "@typescript-eslint/no-explicit-any": "off" 14 | } 15 | } -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { DataPipe } from './data-pipe'; 2 | 3 | export * from './types'; 4 | 5 | /** 6 | * Data Pipeline factory function what creates DataPipe 7 | * @param data Initial array 8 | * 9 | * @example 10 | * dataPipe([1, 2, 3]) 11 | */ 12 | export function dataPipe(data?: T[]): DataPipe { 13 | data = data || []; 14 | return new DataPipe(data); 15 | } 16 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "chrome", 9 | "request": "launch", 10 | "name": "Launch Chrome against localhost", 11 | "url": "http://localhost:10001/", 12 | "webRoot": "${workspaceFolder}" 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /src/examples/esm/utils.mjs: -------------------------------------------------------------------------------- 1 | import { parseDatetimeOrNull, parseNumberOrNull, toTable, fromTable } from 'datapipe-js/utils'; 2 | 3 | console.log(parseDatetimeOrNull('10-10-2019'), parseNumberOrNull('0.45')); 4 | 5 | const data = [ 6 | { name: "John", country: "US"}, { name: "Joe", country: "US"}, { name: "Bill", country: "US"}, { name: "Adam", country: "UK"}, 7 | { name: "Scott", country: "UK"}, { name: "Diana",country: "UK"}, { name: "Marry",country: "FR"}, { name: "Luc",country: "FR"} 8 | ] 9 | 10 | const tableData = toTable(data); 11 | console.log('Table format data: ', tableData); 12 | console.log('Objects list format: ', fromTable(tableData)); 13 | -------------------------------------------------------------------------------- /src/examples/cjs/utils.js: -------------------------------------------------------------------------------- 1 | const { parseDatetimeOrNull, parseNumberOrNull, toTable, fromTable } = require('datapipe-js/utils'); 2 | 3 | console.log(parseDatetimeOrNull('10-10-2019'), parseNumberOrNull('0.45')); 4 | 5 | const data = [ 6 | { name: "John", country: "US"}, { name: "Joe", country: "US"}, { name: "Bill", country: "US"}, { name: "Adam", country: "UK"}, 7 | { name: "Scott", country: "UK"}, { name: "Diana",country: "UK"}, { name: "Marry",country: "FR"}, { name: "Luc",country: "FR"} 8 | ] 9 | 10 | const tableData = toTable(data); 11 | console.log('Table format data: ', tableData); 12 | console.log('Objects list format: ', fromTable(tableData)); 13 | -------------------------------------------------------------------------------- /rollup.config.dev.js: -------------------------------------------------------------------------------- 1 | import typescript from 'rollup-plugin-typescript2'; 2 | import serve from 'rollup-plugin-serve' 3 | import livereload from 'rollup-plugin-livereload' 4 | 5 | export default { 6 | 7 | input: 'src/index.ts', 8 | output: { 9 | name: 'dp', 10 | file: 'dist/data-pipe.min.js', 11 | format: 'umd', 12 | sourcemap: true, 13 | globals: {} 14 | }, 15 | external: [], 16 | plugins: [ 17 | typescript({ 18 | abortOnError: false 19 | }), 20 | serve({contentBase: '', open: true}), 21 | livereload('dist') 22 | ], 23 | watch: { 24 | exclude: ['node_modules/**'], 25 | include: 'src/**' 26 | } 27 | }; 28 | -------------------------------------------------------------------------------- /src/examples/esm/array.mjs: -------------------------------------------------------------------------------- 1 | import { groupBy, first, select, where, count } from 'datapipe-js/array'; 2 | 3 | const data = [ 4 | { name: "John", country: "US"}, { name: "Joe", country: "US"}, { name: "Bill", country: "US"}, { name: "Adam", country: "UK"}, 5 | { name: "Scott", country: "UK"}, { name: "Diana",country: "UK"}, { name: "Marry",country: "FR"}, { name: "Luc",country: "FR"} 6 | ] 7 | 8 | const groups = groupBy(data, i => i.country); 9 | const list = select(groups, g => ({ 10 | country: first(g).country, 11 | names: select(g, r => r.name).join(", "), 12 | count: count(g) 13 | })); 14 | 15 | const summaryForUS = where(list, r => r.country === "US"); 16 | 17 | console.log(summaryForUS); 18 | -------------------------------------------------------------------------------- /src/examples/cjs/array.js: -------------------------------------------------------------------------------- 1 | const { groupBy, first, select, where, count } = require('datapipe-js/array'); 2 | 3 | const data = [ 4 | { name: "John", country: "US"}, { name: "Joe", country: "US"}, { name: "Bill", country: "US"}, { name: "Adam", country: "UK"}, 5 | { name: "Scott", country: "UK"}, { name: "Diana",country: "UK"}, { name: "Marry",country: "FR"}, { name: "Luc",country: "FR"} 6 | ] 7 | 8 | const groups = groupBy(data, i => i.country); 9 | const list = select(groups, g => ({ 10 | country: first(g).country, 11 | names: select(g, r => r.name).join(", "), 12 | count: count(g) 13 | })); 14 | 15 | const summaryForUS = where(list, r => r.country === "US"); 16 | 17 | console.log(summaryForUS); 18 | -------------------------------------------------------------------------------- /src/examples/esm/datapipe.mjs: -------------------------------------------------------------------------------- 1 | import { dataPipe } from 'datapipe-js'; 2 | 3 | const data = [ 4 | { name: "John", country: "US"}, { name: "Joe", country: "US"}, { name: "Bill", country: "US"}, { name: "Adam", country: "UK"}, 5 | { name: "Scott", country: "UK"}, { name: "Diana",country: "UK"}, { name: "Marry",country: "FR"}, { name: "Luc",country: "FR"} 6 | ] 7 | 8 | const summaryForUS = dataPipe(data) 9 | .groupBy(i => i.country) 10 | .select(g => { 11 | const r = {} 12 | r.country = dataPipe(g).first().country 13 | r.names = dataPipe(g).map(r => r.name).toArray().join(", ") 14 | r.count = dataPipe(g).count() 15 | return r; 16 | }) 17 | .where(r => r.country === "US") 18 | .toArray() 19 | 20 | console.log(summaryForUS); 21 | -------------------------------------------------------------------------------- /src/examples/cjs/datapipe.js: -------------------------------------------------------------------------------- 1 | const { dataPipe } = require('datapipe-js'); 2 | 3 | const data = [ 4 | { name: "John", country: "US"}, { name: "Joe", country: "US"}, { name: "Bill", country: "US"}, { name: "Adam", country: "UK"}, 5 | { name: "Scott", country: "UK"}, { name: "Diana",country: "UK"}, { name: "Marry",country: "FR"}, { name: "Luc",country: "FR"} 6 | ] 7 | 8 | const summaryForUS = dataPipe(data) 9 | .groupBy(i => i.country) 10 | .select(g => { 11 | const r = {} 12 | r.country = dataPipe(g).first().country 13 | r.names = dataPipe(g).map(r => r.name).toArray().join(", ") 14 | r.count = dataPipe(g).count() 15 | return r; 16 | }) 17 | .where(r => r.country === "US") 18 | .toArray() 19 | 20 | console.log(summaryForUS); 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # compiled output 4 | /dist 5 | /tmp 6 | /out-tsc 7 | /release 8 | /lib 9 | /docs 10 | /md-docs 11 | /array 12 | /string 13 | /utils 14 | 15 | # dependencies 16 | /node_modules 17 | 18 | # IDEs and editors 19 | /.idea 20 | .project 21 | .classpath 22 | .c9/ 23 | *.launch 24 | .settings/ 25 | *.sublime-workspace 26 | 27 | # IDE - VSCode 28 | .vscode/* 29 | !.vscode/settings.json 30 | !.vscode/tasks.json 31 | !.vscode/launch.json 32 | !.vscode/extensions.json 33 | 34 | # misc 35 | /.sass-cache 36 | /connect.lock 37 | /coverage 38 | /libpeerconnection.log 39 | npm-debug.log 40 | yarn-error.log 41 | testem.log 42 | /typings 43 | 44 | # System Files 45 | .DS_Store 46 | Thumbs.db 47 | 48 | package-lock.json 49 | *.tgz 50 | debug.log 51 | -------------------------------------------------------------------------------- /src/examples/test.js: -------------------------------------------------------------------------------- 1 | const { dataPipe } = require('datapipe-js'); 2 | const { avg } = require('datapipe-js/array'); 3 | const fetch = require('node-fetch'); 4 | 5 | async function main() { 6 | 7 | const dataUrl = "https://raw.githubusercontent.com/FalconSoft/sample-data/master/CSV/sample-testing-data-100.csv"; 8 | const csv = await (await fetch(dataUrl)).text(); 9 | 10 | return dataPipe() 11 | .fromCsv(csv) 12 | .groupBy(r => r.Country) 13 | .select(g => ({ 14 | country: dataPipe(g).first().Country, 15 | sales: dataPipe(g).sum(i => i.Sales), 16 | avg: avg(g, i => i.Sales), 17 | count: g.length 18 | }) 19 | ) 20 | .where(r => r.sales > 5000) 21 | .sort("sales DESC") 22 | .toArray(); 23 | } 24 | 25 | main() 26 | .then(console.log) 27 | .catch(console.error) 28 | -------------------------------------------------------------------------------- /src/_internals.ts: -------------------------------------------------------------------------------- 1 | import { Selector } from './types'; 2 | import { dateToString } from './utils'; 3 | 4 | export function fieldSelector( 5 | input: string | string[] | Selector 6 | ): Selector { 7 | if (typeof input === 'function') { 8 | return (item: T): string => { 9 | const value = input(item) as unknown; 10 | return value instanceof Date ? dateToString(value as Date) : String(value); 11 | }; 12 | } else if (typeof input === 'string') { 13 | return (item: T): string => { 14 | const value = (item as Record)[input]; 15 | return value instanceof Date ? dateToString(value) : String(value); 16 | }; 17 | } else if (Array.isArray(input)) { 18 | return (item: T): string => input.map(r => (item as Record)[r]).join('|'); 19 | } else { 20 | throw Error(`Unknown input. Can't create a fieldSelector`); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/tests/table.spec.ts: -------------------------------------------------------------------------------- 1 | import { data } from './array.spec'; 2 | import { fromTable, toTable } from '../utils'; 3 | import { ScalarObject } from '../types'; 4 | 5 | export const table = { 6 | fields: ['name', 'country'], 7 | rows: [ 8 | ['John', 'US'], 9 | ['Joe', 'US'], 10 | ['Bill', 'US'], 11 | ['Adam', 'UK'], 12 | ['Scott', 'UK'], 13 | ['Diana', 'UK'], 14 | ['Marry', 'FR'], 15 | ['Luc', 'FR'] 16 | ] 17 | }; 18 | 19 | describe('Test table methods', () => { 20 | it('fromTable', () => { 21 | const list = fromTable(table.rows, table.fields); 22 | expect(list[0]).toHaveProperty('name'); 23 | expect(list[0]).toHaveProperty('country'); 24 | expect(list.length).toBe(table.rows.length); 25 | }); 26 | 27 | it('toTable', () => { 28 | const tableData = toTable(data as ScalarObject[]); 29 | expect(tableData).toHaveProperty('rows'); 30 | expect(tableData).toHaveProperty('fieldNames'); 31 | expect(tableData.rows.length).toBe(data.length); 32 | expect(tableData.fieldNames[0]).toBe('name'); 33 | expect(tableData.fieldNames[1]).toBe('country'); 34 | }); 35 | }); 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 FalconSoft Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Data Pipe JS 5 | 6 | 7 | 8 | 9 | 10 |

Data Pipe testing page. Do not expect anything here!

11 | 12 | 13 | 14 | 15 | 16 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /src/tests/data-pipe.spec.ts: -------------------------------------------------------------------------------- 1 | import { dataPipe } from '../index'; 2 | import { table } from './table.spec'; 3 | import { data } from './array.spec'; 4 | import { DataPipe } from '../data-pipe'; 5 | 6 | describe('DataPipe specification', () => { 7 | it('dataPipe returns DataPipe', () => { 8 | expect(dataPipe([]) instanceof DataPipe).toBeTruthy(); 9 | }); 10 | 11 | it('toArray', () => { 12 | const arr = dataPipe(['US']).toArray(); 13 | expect(arr instanceof Array).toBeTruthy(); 14 | expect(arr).toEqual(['US']); 15 | }); 16 | 17 | it('select/map', () => { 18 | const arr = [{ country: 'US' }]; 19 | expect( 20 | dataPipe(arr) 21 | .select(i => i.country) 22 | .toArray() 23 | ).toEqual(['US']); 24 | expect( 25 | dataPipe(arr) 26 | .map(i => i.country) 27 | .toArray() 28 | ).toEqual(['US']); 29 | }); 30 | 31 | it('groupBy', () => { 32 | const dp = dataPipe(data).groupBy(i => i.country); 33 | expect(dp.toArray().length).toBe(3); 34 | }); 35 | 36 | it('fromTable/toTable', () => { 37 | const tData = dataPipe() 38 | .fromTable(table.rows, table.fields) 39 | .filter(r => r.country !== 'US') 40 | .toTable(); 41 | expect(tData.rows.length).toBe(5); 42 | }); 43 | 44 | it('unique', () => { 45 | const arr = [1, '5', 3, 5, 3, 4, 3, 1]; 46 | expect(dataPipe(arr).unique().toArray().length).toBe(5); 47 | }); 48 | }); 49 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import typescript from 'rollup-plugin-typescript2'; 2 | import { uglify } from 'rollup-plugin-uglify'; 3 | import del from 'rollup-plugin-delete' 4 | 5 | const pkg = require('./package.json'); 6 | const input = 'src/index.ts'; 7 | const subModules = ['array', 'string', 'utils']; 8 | 9 | export default [{ 10 | input: 'src/commonjs.ts', 11 | output: [ 12 | { file: 'dist/data-pipe.min.js', name: 'dp', format: 'umd', sourcemap: true, compact: true }, 13 | ], 14 | treeshake: true, 15 | plugins: [ 16 | del({ 17 | targets: ['./dist', ...subModules.map(m => `./${m}`)], 18 | hook: 'buildStart', 19 | runOnce: true 20 | }), 21 | typescript({ 22 | clean: true 23 | }), 24 | uglify() 25 | ] 26 | }, { 27 | input: 'src/commonjs.ts', 28 | output: { file: pkg.main, format: 'cjs', sourcemap: true, compact: true }, 29 | treeshake: true, 30 | plugins: [ 31 | typescript({ 32 | clean: true 33 | }) 34 | ] 35 | }, { 36 | input: input, 37 | output: { file: pkg.module, format: 'esm', sourcemap: true, compact: true }, 38 | treeshake: true, 39 | plugins: [ 40 | typescript({ 41 | clean: true 42 | }) 43 | ] 44 | }, ...subModules.map(subFolder => ({ 45 | input: `src/${subFolder}/index.ts`, 46 | output: { file: `${subFolder}/index.mjs`, format: 'esm', sourcemap: true, compact: true }, 47 | treeshake: true, 48 | plugins: [ 49 | typescript({ 50 | clean: true 51 | }) 52 | ], 53 | })), ...subModules.map(subFolder => ({ 54 | input: `src/${subFolder}/index.ts`, 55 | output: { file: `${subFolder}/index.js`, format: 'cjs', sourcemap: true, compact: true }, 56 | treeshake: true, 57 | plugins: [ 58 | typescript({ 59 | clean: true 60 | }) 61 | ] 62 | }))]; 63 | -------------------------------------------------------------------------------- /src/tests/string-utils.spec.ts: -------------------------------------------------------------------------------- 1 | import { trimStart, trimEnd, split } from '../string'; 2 | 3 | describe('string utils', () => { 4 | it('ltrim', () => { 5 | expect(trimStart('.net', '.')).toBe('net'); 6 | expect(trimStart('...net', '.')).toBe('net'); 7 | expect(trimStart('.+net', '.+')).toBe('net'); 8 | }); 9 | 10 | it('rtrim', () => { 11 | expect(trimEnd('net.', '.')).toBe('net'); 12 | expect(trimEnd('net..', '.')).toBe('net'); 13 | expect(trimEnd('net++', '.+')).toBe('net'); 14 | }); 15 | }); 16 | 17 | describe('string split', () => { 18 | it('split => simple', () => { 19 | const t = split('a,b', ','); 20 | expect(t.length).toBe(2); 21 | expect(t[0]).toBe('a'); 22 | expect(t[1]).toBe('b'); 23 | }); 24 | 25 | it('split => simple2', () => { 26 | const t = split('a2,b2', ','); 27 | expect(t.length).toBe(2); 28 | expect(t[0]).toBe('a2'); 29 | expect(t[1]).toBe('b2'); 30 | }); 31 | 32 | it('split => with brackets 1', () => { 33 | const t = split('a(s,d)t,b2', ',', ['()']); 34 | expect(t.length).toBe(2); 35 | expect(t[0]).toBe('a(s,d)t'); 36 | expect(t[1]).toBe('b2'); 37 | }); 38 | 39 | it('split => with brackets 2', () => { 40 | const t = split('a(s,d),b2', ',', ['()']); 41 | expect(t.length).toBe(2); 42 | expect(t[0]).toBe('a(s,d)'); 43 | expect(t[1]).toBe('b2'); 44 | }); 45 | 46 | it('split => with brackets 3', () => { 47 | const t = split('a(s,ff(e,r)d),b2', ',', ['()']); 48 | expect(t.length).toBe(2); 49 | expect(t[0]).toBe('a(s,ff(e,r)d)'); 50 | expect(t[1]).toBe('b2'); 51 | }); 52 | 53 | it('split => with brackets 4', () => { 54 | const t = split('a(s,ff(e,r)d),b2ff(e,r)', ',', ['()']); 55 | expect(t.length).toBe(2); 56 | expect(t[0]).toBe('a(s,ff(e,r)d)'); 57 | expect(t[1]).toBe('b2ff(e,r)'); 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /src/utils/json-parser.ts: -------------------------------------------------------------------------------- 1 | import { trim } from '../string'; 2 | 3 | export class JSONParser { 4 | private readonly whitespaces = ' \n\r\t'; 5 | private ignoreWhiteSpace = false; 6 | private token = ''; 7 | private openObjectCount = 0; 8 | private openArrayCount = 0; 9 | private count = 0; 10 | private firstChar = ''; 11 | 12 | private isString = false; 13 | private isKeyCollector = false; 14 | private prevChar = ''; 15 | private itemKey = ''; 16 | private isObjectMap = false; 17 | 18 | processJsonItems(ch: string, processCallback: (obj: string, key: string | number) => void): void { 19 | if (!this.firstChar) { 20 | // ignore begining whitespaces 21 | if (!ch.trim().length) { 22 | return; 23 | } 24 | this.firstChar = ch; 25 | if (this.firstChar === '{') { 26 | this.isKeyCollector = true; 27 | this.isObjectMap = true; 28 | } 29 | return; 30 | } 31 | 32 | if (this.ignoreWhiteSpace && this.whitespaces.indexOf(ch) >= 0) { 33 | return; 34 | } 35 | 36 | if (!this.isString && ch === '{') { 37 | this.openObjectCount++; 38 | } else if (!this.isString && ch === '}') { 39 | this.openObjectCount--; 40 | } else if (!this.isString && ch === '[') { 41 | this.openArrayCount++; 42 | } else if (!this.isString && ch === ']') { 43 | this.openArrayCount--; 44 | } else if (!this.isString && this.isKeyCollector && ch === ':') { 45 | this.itemKey = this.token; 46 | this.token = ''; 47 | this.isKeyCollector = false; 48 | return; 49 | } else if (ch === '"' && (this.prevChar !== '\\' || this.token.endsWith('\\\\'))) { 50 | this.isString = !this.isString; 51 | } 52 | 53 | const objectDone = 54 | !this.isKeyCollector && this.openArrayCount === 0 && this.openObjectCount === 0; 55 | 56 | if (objectDone && ch === ',') { 57 | return; 58 | } 59 | 60 | this.token += ch; 61 | this.prevChar = ch; 62 | 63 | if (objectDone && this.token.trim()) { 64 | const key = this.itemKey ? trim(this.itemKey, `\n\t ,"'`) : this.count; 65 | processCallback(this.token, key); 66 | 67 | if (this.isObjectMap) { 68 | this.isKeyCollector = true; 69 | } 70 | 71 | this.count++; 72 | this.itemKey = ''; 73 | this.token = ''; 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | export type Predicate = (p: T) => boolean; 2 | export type Selector = (p: T) => V; 3 | export class ParsingOptions { 4 | delimiter = ','; 5 | skipRows = 0; 6 | keepOriginalHeaders = false; 7 | textFields: string[] = []; 8 | dateFields: (string | string[])[] = []; 9 | numberFields: string[] = []; 10 | booleanFields: string[] = []; 11 | skipUntil?: (tokens: string[]) => boolean; 12 | takeWhile?: (tokens: string[]) => boolean; 13 | parseFields?: Record; 14 | elementSelector?: (fieldDescriptions: FieldDescription[], tokens: string[]) => unknown; 15 | } 16 | 17 | export type PrimitiveType = string | number | bigint | boolean | null; 18 | 19 | /** 20 | * ScalarType represent a single value types what includes Date and can be null 21 | */ 22 | export type ScalarType = PrimitiveType | Date; 23 | 24 | /** 25 | * Scalar object is a simple object where key is always string and value could be a ScalarType only 26 | * Scalar type includes Date object 27 | */ 28 | export type ScalarObject = Record; 29 | 30 | /** 31 | * PrimitivesObject is a simple object where key is a string and value can be Primitive 32 | * This object can be transfered with REST call. Doesn'r have a Date 33 | */ 34 | export type PrimitivesObject = Record; 35 | 36 | /** 37 | * Commonly used and recognized types 38 | */ 39 | export enum DataTypeName { 40 | String = 'String', 41 | LargeString = 'LargeString', 42 | WholeNumber = 'WholeNumber', 43 | BigIntNumber = 'BigIntNumber', 44 | FloatNumber = 'FloatNumber', 45 | DateTime = 'DateTime', 46 | Date = 'Date', 47 | Boolean = 'Boolean' 48 | } 49 | 50 | export interface Table { 51 | fieldDataTypes?: DataTypeName[]; 52 | fieldNames: string[]; 53 | rows: T[][]; 54 | } 55 | 56 | /** 57 | * A simple data table structure what provides a most efficient way 58 | * to send data across the wire 59 | */ 60 | export type TableDto = Table; 61 | 62 | export type ScallarTable = Table; 63 | 64 | export interface StringsDataTable extends Table { 65 | fieldDescriptions: FieldDescription[]; 66 | } 67 | 68 | export interface FieldDescription { 69 | index: number; 70 | fieldName: string; 71 | isNullable: boolean; 72 | isUnique: boolean; 73 | isObject: boolean; 74 | maxSize?: number; 75 | dataTypeName?: DataTypeName; 76 | } 77 | -------------------------------------------------------------------------------- /src/utils/table.ts: -------------------------------------------------------------------------------- 1 | import { ScalarObject, PrimitiveType, TableDto, DataTypeName } from '../types'; 2 | import { parseDatetimeOrNull, dateToString } from './helpers'; 3 | 4 | /** 5 | * Get JSON type array for tabel type array. 6 | * @param rowsOrTable Table data or Array of values . 7 | * @param fieldNames Column names. If not provided then, it will be auto generated 8 | * @param fieldDataTypes Column names 9 | */ 10 | export function fromTable( 11 | rowsOrTable: PrimitiveType[][] | TableDto, 12 | fieldNames?: string[], 13 | fieldDataTypes?: DataTypeName[] 14 | ): ScalarObject[] { 15 | const table = rowsOrTable as TableDto; 16 | const rows = table?.rows || rowsOrTable || []; 17 | 18 | if (!rows.length) { 19 | // return empty array 20 | return []; 21 | } 22 | 23 | fieldNames = table?.fieldNames || fieldNames || rows[0].map((v, i) => `Field${i}`); 24 | fieldDataTypes = table?.fieldDataTypes || fieldDataTypes || []; 25 | 26 | const values: ScalarObject[] = []; 27 | for (const row of rows) { 28 | const value: ScalarObject = {}; 29 | for (let i = 0, len = fieldNames.length; i < len; i++) { 30 | const fieldName = fieldNames[i]; 31 | const dataType = fieldDataTypes.length ? fieldDataTypes[i] : null; 32 | value[fieldName] = 33 | (dataType === DataTypeName.DateTime || dataType === DataTypeName.Date) && row[i] 34 | ? parseDatetimeOrNull(row[i] as string | Date) 35 | : row[i]; 36 | } 37 | values.push(value); 38 | } 39 | 40 | return values; 41 | } 42 | 43 | /** 44 | * Gets table data from JSON type array. 45 | * @param array 46 | * @param rowsFieldName 47 | * @param fieldsFieldName 48 | */ 49 | export function toTable(values: ScalarObject[]): TableDto { 50 | const tableDto: TableDto = { 51 | fieldNames: [], 52 | rows: [] 53 | }; 54 | 55 | if (!values?.length) { 56 | return tableDto; 57 | } 58 | 59 | const fN = new Set(); 60 | values.forEach(v => { 61 | Object.keys(v).forEach(k => fN.add(k)); 62 | }); 63 | 64 | tableDto.fieldNames = Array.from(fN.values()); 65 | tableDto.rows = values.map(rowValues => { 66 | return tableDto.fieldNames.reduce((r, field) => { 67 | const v = rowValues[field]; 68 | const val = v instanceof Date ? dateToString(v) : v; 69 | r.push(val as PrimitiveType); 70 | return r; 71 | }, [] as PrimitiveType[]); 72 | }); 73 | return tableDto; 74 | } 75 | -------------------------------------------------------------------------------- /src/tests/json-parser.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-useless-escape */ 2 | import { JSONParser } from '../utils'; 3 | 4 | describe('Json Parser specification', () => { 5 | function jsonProcessor(text: string): Record[] { 6 | const result: Record[] = []; 7 | const parser = new JSONParser(); 8 | for (const ch of text) { 9 | parser.processJsonItems(ch, (itemText: string, key: string | number) => { 10 | let item = JSON.parse(itemText); 11 | 12 | if (typeof key === 'string') { 13 | item = { key, item }; 14 | } 15 | 16 | result.push(item); 17 | }); 18 | } 19 | 20 | return result; 21 | } 22 | 23 | it('Simple JSON', () => { 24 | const result = jsonProcessor('[{"value":"test value"},{"value":2},{"value":3}]'); 25 | 26 | expect(result.length).toBe(3); 27 | expect((result[0] as Record).value).toBe('test value'); 28 | expect((result[1] as Record).value).toBe(2); 29 | }); 30 | 31 | // it('Simple JSV', () => { 32 | // const result = JSONParser.parseJson('[{value:test value},{value:2},{value:3}]'); 33 | 34 | // expect(result.length).toBe(3); 35 | // expect(result[0].value).toBe('test value'); 36 | // expect(result[1].value).toBe(2); 37 | // }); 38 | 39 | it('JSV with space', () => { 40 | const result = jsonProcessor('[{"value":"test"},{"value":2, "value1": 21},{"value":3}]'); 41 | 42 | expect(result.length).toBe(3); 43 | expect((result[1] as Record).value1).toBe(21); 44 | }); 45 | 46 | it('json inner object', () => { 47 | const result = jsonProcessor( 48 | '[{"v":"test", "obj":{"v1":2, "v2": 21},"value":3}, {"v":"test2", "obj":{"v1":22, "v2": 212},"value":32}]' 49 | ); 50 | 51 | expect(result.length).toBe(2); 52 | expect(((result[1] as Record).obj as Record).v2).toBe(212); 53 | }); 54 | 55 | it('JSON simple array', () => { 56 | const result = jsonProcessor('[{"v":"test", "arr":[{"v1":1}, {"v1":2}]}]'); 57 | 58 | expect(result.length).toBe(1); 59 | expect((((result[0] as Record).arr as unknown[])[1] as Record).v1).toBe(2); 60 | }); 61 | 62 | 63 | it('Simple JSON with " to escape', () => { 64 | const result = jsonProcessor('[{"value":"test \\"value"}]'); 65 | expect((result[0] as Record).value).toBe('test "value'); 66 | }); 67 | 68 | it('Simple JSON with " to escape in the end', () => { 69 | const result = jsonProcessor('[{"value":"test value\\""}]'); 70 | expect((result[0] as Record).value).toBe('test value\"'); 71 | }); 72 | 73 | it('Simple JSON with \\ to escape in the end', () => { 74 | const result = jsonProcessor(`[{"value":"test value\\\\d"}]`); 75 | expect((result[0] as Record).value).toBe('test value\\d'); 76 | }); 77 | 78 | // 79 | // 80 | }); 81 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "datapipe-js", 3 | "version": "0.3.31", 4 | "description": "dataPipe is a data processing and data analytics library for JavaScript. Inspired by LINQ (C#) and Pandas (Python)", 5 | "main": "dist/cjs/data-pipe.js", 6 | "module": "dist/esm/data-pipe.mjs", 7 | "typings": "dist/index.d.ts", 8 | "exports": { 9 | ".": { 10 | "import": "./dist/esm/data-pipe.mjs", 11 | "require": "./dist/cjs/data-pipe.js" 12 | }, 13 | "./array": { 14 | "import": "./array/index.mjs", 15 | "require": "./array/index.js" 16 | }, 17 | "./string": { 18 | "import": "./string/index.mjs", 19 | "require": "./string/index.js" 20 | }, 21 | "./utils": { 22 | "import": "./utils/index.mjs", 23 | "require": "./utils/index.js" 24 | } 25 | }, 26 | "scripts": { 27 | "test": "jest", 28 | "test:dev": "jest --watch", 29 | "test:dev:debug": "node --inspect-brk node_modules/.bin/jest --runInBand --watch", 30 | "build": "npx rollup -c && node copy-types.js", 31 | "build:publish": "npm run build && npm publish", 32 | "deploy": "npm run docs && npx gh-pages -d docs", 33 | "dev": "npx rollup --config rollup.config.dev.js --watch", 34 | "lint": "npx eslint . --ext .ts", 35 | "lint-fix": "eslint . --ext .ts --fix" 36 | }, 37 | "repository": { 38 | "type": "git", 39 | "url": "git+https://github.com/falconsoft/dataPipe.git" 40 | }, 41 | "author": "Pavlo Paska - ppaska@falconsoft-ltd.com", 42 | "license": "MIT", 43 | "bugs": { 44 | "url": "https://github.com/falconsoft/dataPipe/issues" 45 | }, 46 | "homepage": "https://www.datapipe-js.com/", 47 | "keywords": [ 48 | "data", 49 | "data-analysis", 50 | "LINQ", 51 | "data-wrangling", 52 | "pandas", 53 | "data-management", 54 | "data-science", 55 | "data-manipulation", 56 | "json", 57 | "data-mungling", 58 | "data-cleaning", 59 | "data-clensing" 60 | ], 61 | "devDependencies": { 62 | "@types/jest": "^24.9.1", 63 | "@typescript-eslint/eslint-plugin": "^2.24.0", 64 | "@typescript-eslint/parser": "^2.24.0", 65 | "eslint": "^6.8.0", 66 | "husky": "^4.2.5", 67 | "jest": "^24.9.0", 68 | "jest-fetch-mock": "^2.1.2", 69 | "rollup": "^1.31.1", 70 | "rollup-plugin-copy": "^3.3.0", 71 | "rollup-plugin-delete": "^2.0.0", 72 | "rollup-plugin-livereload": "^1.0.4", 73 | "rollup-plugin-serve": "^1.0.1", 74 | "rollup-plugin-typescript2": "^0.25.3", 75 | "rollup-plugin-uglify": "^6.0.4", 76 | "ts-jest": "^24.3.0", 77 | "typedoc": "^0.17.3", 78 | "typedoc-plugin-markdown": "^2.2.17", 79 | "typescript": "^3.8.3" 80 | }, 81 | "husky": { 82 | "hooks": { 83 | "pre-commit": "npm run lint && npm run test" 84 | } 85 | } 86 | } -------------------------------------------------------------------------------- /src/string/stringUtils.ts: -------------------------------------------------------------------------------- 1 | export function formatCamelStr(str = ''): string { 2 | return str 3 | .replace(/^\w/, c => c.toUpperCase()) 4 | .replace(/([a-z0-9])([A-Z])/g, '$1 $2') 5 | .replace(/_/g, ' '); 6 | } 7 | 8 | export function replaceAll(text: string, searchValue: string, replaceValue = ''): string { 9 | return text.replace(new RegExp(searchValue, 'g'), replaceValue || ''); 10 | } 11 | 12 | /** 13 | * trims characters from the left 14 | * @param text text to trim 15 | * @param characters character to trim 16 | * @returns trimmed string 17 | */ 18 | export function trimStart(text: string, characters = ' \n\t\r'): string { 19 | if (!text) { 20 | return text; 21 | } 22 | 23 | let startIndex = 0; 24 | while (characters.indexOf(text.charAt(startIndex)) >= 0) { 25 | startIndex++; 26 | } 27 | return text.substring(startIndex); 28 | } 29 | 30 | /** 31 | * trims characters from the right 32 | * @param text text to trim 33 | * @param characters character to trim 34 | * @returns trimmed string 35 | */ 36 | export function trimEnd(text: string, characters = ' \n\t\r'): string { 37 | if (!text) { 38 | return text; 39 | } 40 | 41 | let endIndex = text.length; 42 | while (characters.indexOf(text.charAt(endIndex - 1)) >= 0) { 43 | endIndex--; 44 | } 45 | return text.substring(0, endIndex); 46 | } 47 | 48 | /** 49 | * trims characters from both sides 50 | * @param text text to trim 51 | * @param characters character to trim 52 | * @returns trimmed string 53 | */ 54 | export function trim(text: string, characters = ' \n\t\r'): string { 55 | return trimStart(trimEnd(text, characters), characters); 56 | } 57 | 58 | /** 59 | * Splits string into array of tokens based on a separator(one or many). 60 | * Also, you can define open close brackets. 61 | * e.g. split('field1=func(a,b,c),field2=4', ',', ['()']) 62 | * // result = ["field1=func(a,b,c)", "field2=4"] 63 | * @param text Text to split 64 | * @param separator 65 | * @param openClose 66 | * @returns 67 | */ 68 | export function split(text: string, separator = ',', openClose?: string[]): string[] { 69 | const res: string[] = []; 70 | 71 | if (!text) { 72 | return res; 73 | } 74 | 75 | openClose = openClose || []; 76 | let index = -1; 77 | 78 | let token = ''; 79 | while (++index < text.length) { 80 | let currentChar = text[index]; 81 | 82 | const oIndex = openClose.findIndex(s => s[0] === currentChar); 83 | if (oIndex >= 0) { 84 | token += text[index]; 85 | let innerBrackets = 0; 86 | while (text[++index] !== openClose[oIndex][1] || innerBrackets) { 87 | currentChar = text[index]; 88 | token += currentChar; 89 | 90 | if (currentChar === openClose[oIndex][0]) { 91 | innerBrackets++; 92 | } 93 | 94 | if (currentChar === openClose[oIndex][1]) { 95 | innerBrackets--; 96 | } 97 | 98 | if (index + 1 === text.length) { 99 | throw new Error(`Closing bracket is missing`); 100 | } 101 | } 102 | token += text[index]; 103 | continue; 104 | } 105 | 106 | if (separator.includes(currentChar)) { 107 | res.push(token); 108 | token = ''; 109 | } else { 110 | token += currentChar; 111 | } 112 | } 113 | 114 | res.push(token); 115 | 116 | return res; 117 | } 118 | -------------------------------------------------------------------------------- /src/array/utils.ts: -------------------------------------------------------------------------------- 1 | import { parseNumber, parseDatetimeOrNull } from '../utils'; 2 | import { ScalarType, Selector } from '..'; 3 | import { fieldSelector } from '../_internals'; 4 | 5 | function compareStrings(a: string, b: unknown): number { 6 | return a.localeCompare(String(b)); 7 | } 8 | 9 | function compareNumbers(a: number, b: unknown): number { 10 | const bNumVal = parseNumber(b as ScalarType); 11 | if (bNumVal === undefined) { 12 | return 1; 13 | } 14 | 15 | return a - bNumVal; 16 | } 17 | 18 | function compareObjects(a: unknown, b: unknown): number { 19 | const aDate = parseDatetimeOrNull(a as string); 20 | const bDate = parseDatetimeOrNull(b as string); 21 | 22 | if (!aDate && !bDate) { 23 | return 0; 24 | } 25 | 26 | if (!aDate) { 27 | return -1; 28 | } 29 | 30 | if (!bDate) { 31 | return 1; 32 | } 33 | 34 | return aDate.getTime() - bDate.getTime(); 35 | } 36 | 37 | function compare(a: Record, b: Record, { field, asc }: { field: string; asc: boolean }): number { 38 | const valA = a[field]; 39 | const valB = b[field]; 40 | const order = asc ? 1 : -1; 41 | 42 | if (valA !== undefined && valB === undefined) { 43 | return order; 44 | } 45 | 46 | switch (typeof valA) { 47 | case 'number': 48 | return order * compareNumbers(valA, valB); 49 | case 'string': 50 | return order * compareStrings(valA, valB); 51 | case 'object': 52 | return order * compareObjects(valA, valB); 53 | case 'undefined': 54 | return valB === undefined ? 0 : -1 * order; 55 | } 56 | return 0; 57 | } 58 | 59 | function comparator(sortFields: Array<{ field: string; asc: boolean }>): (a: Record, b: Record) => number { 60 | if (sortFields.length) { 61 | return (a: Record, b: Record): number => { 62 | for (let i = 0, len = sortFields.length; i < len; i++) { 63 | const res = compare(a, b, sortFields[i]); 64 | 65 | if (res !== 0) { 66 | return res; 67 | } 68 | } 69 | return 0; 70 | }; 71 | } 72 | 73 | return (): number => 0; 74 | } 75 | 76 | /** 77 | * Checks if array is empty or null or array at all 78 | * @param array 79 | */ 80 | export function isArrayEmptyOrNull(array: unknown[]): boolean { 81 | return !array || !Array.isArray(array) || !array.length; 82 | } 83 | 84 | /** 85 | * A simple sort array function with a convenient interface 86 | * @param array The array to process. 87 | * @param fields sorts order. 88 | * @public 89 | * @example 90 | * sort(array, 'name ASC', 'age DESC'); 91 | */ 92 | export function sort>(array: T[], ...fields: string[]): T[] { 93 | if (!array || !Array.isArray(array)) { 94 | throw Error('Array is not provided'); 95 | } 96 | 97 | if (!fields?.length) { 98 | // just a default sort 99 | return array.sort(); 100 | } 101 | 102 | const sortFields = fields.map(field => { 103 | const asc = !field.endsWith(' DESC'); 104 | return { 105 | asc, 106 | field: field.replace(asc ? /\sASC$/ : /\sDESC$/, '') 107 | }; 108 | }); 109 | 110 | array.sort(comparator(sortFields) as (a: T, b: T) => number); 111 | return array; 112 | } 113 | /** 114 | * Converts array of items to the object map. Where key selector can be defined. 115 | * @param array to be converted 116 | * @param keyField a selector or field name for a property name 117 | */ 118 | export function toObject( 119 | array: T[], 120 | keyField: string | string[] | Selector 121 | ): Record { 122 | const result = {} as Record; 123 | 124 | for (const item of array) { 125 | const key = fieldSelector(keyField)(item); 126 | result[key] = item; 127 | } 128 | 129 | return result; 130 | } 131 | 132 | /** 133 | * Convert array of items to into series array or series record. 134 | * @param array Array to be converted 135 | * @param propertyName optional parameter to define a property to be unpacked. 136 | * If it is string the array with values will be returned, otherwise an object with a list of series map 137 | */ 138 | export function toSeries( 139 | array: Array>, 140 | propertyName?: string | string[] 141 | ): Record | ScalarType[] { 142 | if (!array?.length) { 143 | return {}; 144 | } 145 | 146 | // a single property 147 | if (typeof propertyName == 'string') { 148 | return array.map(r => (r[propertyName] === undefined ? null : (r[propertyName] as ScalarType))); 149 | } 150 | 151 | const seriesRecord: Record = {}; 152 | 153 | const seriesNames = 154 | Array.isArray(propertyName) && propertyName.length ? propertyName : Object.keys(array[0]); 155 | 156 | for (let i = 0; i < array.length; i++) { 157 | for (let j = 0; j < seriesNames.length; j++) { 158 | const seriesName = seriesNames[j]; 159 | if (!seriesRecord[seriesName]) { 160 | seriesRecord[seriesName] = []; 161 | } 162 | const value = array[i][seriesName]; 163 | seriesRecord[seriesName].push(value === undefined ? null : (value as ScalarType)); 164 | } 165 | } 166 | 167 | return seriesRecord; 168 | } 169 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | /* Basic Options */ 4 | // "incremental": true, /* Enable incremental compilation */ 5 | "target": "es2015", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */ 6 | "module": "es2015", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ 7 | // "lib": [], /* Specify library files to be included in the compilation. */ 8 | // "allowJs": true, /* Allow javascript files to be compiled. */ 9 | // "checkJs": true, /* Report errors in .js files. */ 10 | // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ 11 | "declaration": true, /* Generates corresponding '.d.ts' file. */ 12 | "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ 13 | "sourceMap": true, /* Generates corresponding '.map' file. */ 14 | // "outFile": "./", /* Concatenate and emit output to single file. */ 15 | // "outDir": "./", /* Redirect output structure to the directory. */ 16 | // "rootDir": "./", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ 17 | // "composite": true, /* Enable project compilation */ 18 | // "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */ 19 | // "removeComments": true, /* Do not emit comments to output. */ 20 | // "noEmit": true, /* Do not emit outputs. */ 21 | // "importHelpers": true, /* Import emit helpers from 'tslib'. */ 22 | // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ 23 | // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ 24 | 25 | /* Strict Type-Checking Options */ 26 | "strict": true, /* Enable all strict type-checking options. */ 27 | // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ 28 | // "strictNullChecks": true, /* Enable strict null checks. */ 29 | // "strictFunctionTypes": true, /* Enable strict checking of function types. */ 30 | // "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */ 31 | // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ 32 | // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ 33 | // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ 34 | 35 | /* Additional Checks */ 36 | // "noUnusedLocals": true, /* Report errors on unused locals. */ 37 | // "noUnusedParameters": true, /* Report errors on unused parameters. */ 38 | // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ 39 | // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ 40 | 41 | /* Module Resolution Options */ 42 | "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ 43 | // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ 44 | // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ 45 | // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ 46 | // "typeRoots": [], /* List of folders to include type definitions from. */ 47 | "types": ["node", "jest"], /* Type declaration files to be included in compilation. */ 48 | // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ 49 | "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ 50 | // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ 51 | // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ 52 | 53 | /* Source Map Options */ 54 | // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ 55 | // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ 56 | // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ 57 | // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ 58 | 59 | /* Experimental Options */ 60 | // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ 61 | // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ 62 | 63 | /* Advanced Options */ 64 | "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */ 65 | }, 66 | "include": [ 67 | "src/**/*" 68 | ], 69 | "exclude": [ 70 | "node_modules", 71 | "**/*.spec.ts" 72 | ], 73 | "typedocOptions": { 74 | "mode": "modules", 75 | "out": "docs", 76 | "excludeExternals": true, 77 | "exclude": "**/*+(index|.spec|.e2e).ts", 78 | "excludeNotExported": false, 79 | "readme": "README.md", 80 | "toc": [ 81 | "data-pipe" 82 | ] 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | // For a detailed explanation regarding each configuration property, visit: 2 | // https://jestjs.io/docs/en/configuration.html 3 | 4 | module.exports = { 5 | // All imported modules in your tests should be mocked automatically 6 | // automock: false, 7 | 8 | // Stop running tests after `n` failures 9 | // bail: 0, 10 | 11 | // Respect "browser" field in package.json when resolving modules 12 | // browser: false, 13 | 14 | // The directory where Jest should store its cached dependency information 15 | // cacheDirectory: "/tmp/jest_rs", 16 | 17 | // Automatically clear mock calls and instances between every test 18 | clearMocks: true, 19 | 20 | // Indicates whether the coverage information should be collected while executing the test 21 | // collectCoverage: false, 22 | 23 | // An array of glob patterns indicating a set of files for which coverage information should be collected 24 | // collectCoverageFrom: null, 25 | 26 | // The directory where Jest should output its coverage files 27 | coverageDirectory: "coverage", 28 | 29 | // An array of regexp pattern strings used to skip coverage collection 30 | // coveragePathIgnorePatterns: [ 31 | // "/node_modules/" 32 | // ], 33 | 34 | // A list of reporter names that Jest uses when writing coverage reports 35 | // coverageReporters: [ 36 | // "json", 37 | // "text", 38 | // "lcov", 39 | // "clover" 40 | // ], 41 | 42 | // An object that configures minimum threshold enforcement for coverage results 43 | // coverageThreshold: null, 44 | 45 | // A path to a custom dependency extractor 46 | // dependencyExtractor: null, 47 | 48 | // Make calling deprecated APIs throw helpful error messages 49 | // errorOnDeprecated: false, 50 | 51 | // Force coverage collection from ignored files using an array of glob patterns 52 | // forceCoverageMatch: [], 53 | 54 | // A path to a module which exports an async function that is triggered once before all test suites 55 | // globalSetup: null, 56 | 57 | // A path to a module which exports an async function that is triggered once after all test suites 58 | // globalTeardown: null, 59 | 60 | // A set of global variables that need to be available in all test environments 61 | globals: { 62 | 'ts-jest': { 63 | // diagnostics: false 64 | } 65 | }, 66 | 67 | // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers. 68 | // maxWorkers: "50%", 69 | 70 | // An array of directory names to be searched recursively up from the requiring module's location 71 | // moduleDirectories: [ 72 | // "node_modules" 73 | // ], 74 | 75 | // An array of file extensions your modules use 76 | // moduleFileExtensions: [ 77 | // "js", 78 | // "json", 79 | // "jsx", 80 | // "ts", 81 | // "tsx", 82 | // "node" 83 | // ], 84 | 85 | // A map from regular expressions to module names that allow to stub out resources with a single module 86 | // moduleNameMapper: {}, 87 | 88 | // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader 89 | // modulePathIgnorePatterns: [], 90 | 91 | // Activates notifications for test results 92 | // notify: false, 93 | 94 | // An enum that specifies notification mode. Requires { notify: true } 95 | // notifyMode: "failure-change", 96 | 97 | // A preset that is used as a base for Jest's configuration 98 | preset: 'ts-jest', 99 | 100 | // Run tests from one or more projects 101 | // projects: null, 102 | 103 | // Use this configuration option to add custom reporters to Jest 104 | // reporters: undefined, 105 | 106 | // Automatically reset mock state between every test 107 | // resetMocks: false, 108 | 109 | // Reset the module registry before running each individual test 110 | // resetModules: false, 111 | 112 | // A path to a custom resolver 113 | // resolver: null, 114 | 115 | // Automatically restore mock state between every test 116 | // restoreMocks: false, 117 | 118 | // The root directory that Jest should scan for tests and modules within 119 | // rootDir: null, 120 | 121 | // A list of paths to directories that Jest should use to search for files in 122 | // roots: [ 123 | // "" 124 | // ], 125 | 126 | // Allows you to use a custom runner instead of Jest's default test runner 127 | // runner: "jest-runner", 128 | 129 | // The paths to modules that run some code to configure or set up the testing environment before each test 130 | setupFiles: ['./setupJest.js'], 131 | 132 | // A list of paths to modules that run some code to configure or set up the testing framework before each test 133 | // setupFilesAfterEnv: [], 134 | 135 | // A list of paths to snapshot serializer modules Jest should use for snapshot testing 136 | // snapshotSerializers: [], 137 | 138 | // The test environment that will be used for testing 139 | // testEnvironment: "jest-environment-jsdom", 140 | 141 | // Options that will be passed to the testEnvironment 142 | // testEnvironmentOptions: {}, 143 | 144 | // Adds a location field to test results 145 | // testLocationInResults: false, 146 | 147 | // The glob patterns Jest uses to detect test files 148 | // testMatch: [ 149 | // "**/__tests__/**/*.[jt]s?(x)", 150 | // "**/?(*.)+(spec|test).[tj]s?(x)" 151 | // ], 152 | 153 | // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped 154 | // testPathIgnorePatterns: [ 155 | // "/node_modules/" 156 | // ], 157 | 158 | // The regexp pattern or array of patterns that Jest uses to detect test files 159 | testRegex: ['\\.spec\\.ts$'], 160 | 161 | // This option allows the use of a custom results processor 162 | // testResultsProcessor: null, 163 | 164 | // This option allows use of a custom test runner 165 | // testRunner: "jasmine2", 166 | 167 | // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href 168 | // testURL: "http://localhost", 169 | 170 | // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout" 171 | // timers: "real", 172 | 173 | // A map from regular expressions to paths to transformers 174 | // transform: null, 175 | 176 | // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation 177 | // transformIgnorePatterns: [ 178 | // "/node_modules/" 179 | // ], 180 | 181 | // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them 182 | // unmockedModulePathPatterns: undefined, 183 | 184 | // Indicates whether each individual test should be reported during the run 185 | // verbose: null, 186 | 187 | // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode 188 | // watchPathIgnorePatterns: [], 189 | 190 | // Whether to use watchman for file crawling 191 | // watchman: true 192 | }; 193 | -------------------------------------------------------------------------------- /src/tests/utils-pipe.spec.ts: -------------------------------------------------------------------------------- 1 | import { 2 | parseDatetimeOrNull, 3 | parseNumberOrNull, 4 | getFieldsInfo, 5 | dateToString, 6 | addBusinessDays 7 | } from '../utils'; 8 | import { FieldDescription, DataTypeName } from '../types'; 9 | 10 | describe('Test dataUtils', () => { 11 | it('parseDate', () => { 12 | expect(parseDatetimeOrNull('')).toBe(null); 13 | expect(parseDatetimeOrNull('10-1010')).toBe(null); 14 | let date = parseDatetimeOrNull('1111'); 15 | if (date) { 16 | expect(date.getFullYear()).toBe(1970); 17 | } 18 | date = parseDatetimeOrNull('10-10-10'); 19 | expect(date).toBeInstanceOf(Date); 20 | if (date) { 21 | expect(date.getMonth()).toBe(9); 22 | } 23 | }); 24 | 25 | it('parseDateTime with miliseconds', () => { 26 | const dt = parseDatetimeOrNull('2020-06-08T13:49:15.16'); 27 | expect(dt).toBeInstanceOf(Date); 28 | expect(dateToString(dt as Date)).toBe('2020-06-08T13:49:15.016Z'); 29 | const strDate = '2020-02-21T13:49:15.967Z'; 30 | expect(dateToString(parseDatetimeOrNull(strDate) as Date)).toBe(strDate); 31 | }); 32 | 33 | it('parseDateTime 1', () => { 34 | const dt = parseDatetimeOrNull('06-Aug-2020'); 35 | expect(dt).toBeInstanceOf(Date); 36 | 37 | const dt2 = parseDatetimeOrNull('8=FIX.4.4^9=58^35=0^49=BuySide^56=SellSide^34=3^52=20190605-12:29:20.259^10=172^'); 38 | expect(dt2).toBeNull(); 39 | }); 40 | 41 | it('parseDateTime with larger miliseconds', () => { 42 | const dt = parseDatetimeOrNull('2020-06-08T13:49:15.16789'); 43 | expect(dt).toBeInstanceOf(Date); 44 | expect(dateToString(dt as Date)).toBe('2020-06-08T13:49:15.167Z'); 45 | const strDate = '2020-02-21T13:49:15.167Z'; 46 | expect(dateToString(parseDatetimeOrNull(strDate) as Date)).toBe(strDate); 47 | }); 48 | 49 | it('parseDateTime with format', () => { 50 | const dt = parseDatetimeOrNull('20200608', 'yyyyMMdd'); 51 | expect(dt).toBeInstanceOf(Date); 52 | expect(dateToString(dt as Date)).toBe('2020-06-08'); 53 | expect(dateToString(parseDatetimeOrNull('202006', 'yyyyMM') as Date)).toBe('2020-06-01'); 54 | expect(dateToString(parseDatetimeOrNull('06/02/2020', 'MM/dd/yyyy') as Date)).toBe( 55 | '2020-06-02' 56 | ); 57 | expect(dateToString(parseDatetimeOrNull('06/02/2020', 'dd/MM/yyyy') as Date)).toBe( 58 | '2020-02-06' 59 | ); 60 | expect(dateToString(parseDatetimeOrNull('2020-06-02', 'yyyy-mm-dd') as Date)).toBe( 61 | '2020-06-02' 62 | ); 63 | }); 64 | 65 | it('last business date', () => { 66 | const dt = parseDatetimeOrNull('20210111', 'yyyyMMdd'); 67 | expect(dt).toBeInstanceOf(Date); 68 | expect(dateToString(dt as Date, 'yyyyMMdd')).toBe('20210111'); 69 | expect(dateToString(addBusinessDays(dt as Date, -1), 'yyyyMMdd')).toBe('20210108'); 70 | }); 71 | 72 | it('parseNumber', () => { 73 | expect(parseNumberOrNull('')).toBe(null); 74 | expect(parseNumberOrNull('11')).toBe(11); 75 | expect(parseNumberOrNull('11.1')).toBe(11.1); 76 | expect(parseNumberOrNull('-11.1')).toBe(-11.1); 77 | expect(parseNumberOrNull(11.1)).toBe(11.1); 78 | expect(parseNumberOrNull(NaN)).toBe(NaN); 79 | }); 80 | 81 | it('getFieldsInfo', () => { 82 | const arr = [2, 4, 5].map(r => ({ val1: r })); 83 | const ff = getFieldsInfo(arr); 84 | expect(ff.length).toBe(1); 85 | expect(ff[0].fieldName).toBe('val1'); 86 | expect(ff[0].dataTypeName).toBe(DataTypeName.WholeNumber); 87 | expect(ff[0].isNullable).toBe(false); 88 | }); 89 | 90 | it('getFieldsInfo2', () => { 91 | const arr = [2, '4', 5].map(r => ({ val1: r })); 92 | const ff = getFieldsInfo(arr); 93 | expect(ff.length).toBe(1); 94 | expect(ff[0].fieldName).toBe('val1'); 95 | expect(ff[0].dataTypeName).toBe(DataTypeName.WholeNumber); 96 | expect(ff[0].isNullable).toBe(false); 97 | }); 98 | 99 | it('getFieldsInfo numbers check', () => { 100 | const mapFn = (r: number | string | null): { val1: number | string | null } => ({ val1: r }); 101 | const fdFn = (arr: (number | string | null)[]): FieldDescription => getFieldsInfo(arr.map(mapFn))[0]; 102 | 103 | expect(fdFn([2, 4, 5]).dataTypeName).toBe(DataTypeName.WholeNumber); 104 | expect(fdFn([2, 4, 5]).isNullable).toBe(false); 105 | expect(fdFn([2, 4, null, 5]).dataTypeName).toBe(DataTypeName.WholeNumber); 106 | expect(fdFn([2, 4, null, 5]).isNullable).toBe(true); 107 | expect(fdFn([2, 4, null, 5]).isObject).toBe(false); 108 | expect(fdFn([2, 4.3, 5]).dataTypeName).toBe(DataTypeName.FloatNumber); 109 | expect(fdFn([2, 4.3, null, 5]).dataTypeName).toBe(DataTypeName.FloatNumber); 110 | expect(fdFn([2, 2147483699, 5]).dataTypeName).toBe(DataTypeName.BigIntNumber); 111 | expect(fdFn([2, 2147483699, null, 5]).dataTypeName).toBe(DataTypeName.BigIntNumber); 112 | expect(fdFn([2, '4', 5]).dataTypeName).toBe(DataTypeName.WholeNumber); 113 | }); 114 | 115 | it('getFieldsInfo DateTime check', () => { 116 | const mapFn = (r: string | Date | number | boolean | null): { val1: string | Date | number | boolean | null } => ({ val1: r }); 117 | const fdFn = (arr: (string | Date | number | boolean | null)[]): FieldDescription => getFieldsInfo(arr.map(mapFn))[0]; 118 | 119 | expect(fdFn(['2019-01-01', '2019-01-02']).dataTypeName).toBe(DataTypeName.Date); 120 | expect(fdFn(['2019-01-01', '2019-01-02']).isNullable).toBe(false); 121 | expect(fdFn(['2019-01-01', null, '2019-01-02']).dataTypeName).toBe(DataTypeName.Date); 122 | expect(fdFn(['2019-01-01', null, '2019-01-02']).isNullable).toBe(true); 123 | expect(fdFn(['2019-01-01', null, '2019-01-02']).isObject).toBe(false); 124 | expect(fdFn([new Date(2001, 1, 1), new Date()]).dataTypeName).toBe(DataTypeName.DateTime); 125 | expect(fdFn([new Date(2001, 1, 1), new Date()]).isObject).toBe(false); 126 | 127 | expect(fdFn(['2019-01-01', 'NOT A DATE', '2019-01-02']).dataTypeName).toBe(DataTypeName.String); 128 | expect(fdFn(['2019-01-01', 76, '2019-01-02']).dataTypeName).toBe(DataTypeName.String); 129 | expect(fdFn(['2019-01-01', 76, false, '2019-01-02']).dataTypeName).toBe(DataTypeName.String); 130 | expect(fdFn(['2019-01-01', 76, false, null, '2019-01-02']).dataTypeName).toBe( 131 | DataTypeName.String 132 | ); 133 | expect(fdFn([new Date(2001, 1, 1), 'NOT A DATE', new Date()]).dataTypeName).toBe( 134 | DataTypeName.String 135 | ); 136 | }); 137 | 138 | it('getFieldsInfo size check', () => { 139 | const mapFn = (r: string | number | null): { val1: string | number | null } => ({ val1: r }); 140 | const fdFn = (arr: (string | number | null)[]): FieldDescription => getFieldsInfo(arr.map(mapFn))[0]; 141 | 142 | const longestText = 'Longest Text'; 143 | expect(fdFn(['Test1', 'Longer', longestText]).maxSize).toBe(longestText.length); 144 | expect(fdFn(['Test1', longestText, 'Longer']).maxSize).toBe(longestText.length); 145 | expect(fdFn(['Test1', longestText, null, 'Longer']).maxSize).toBe(longestText.length); 146 | expect(fdFn([87, longestText, '2019-01-01']).maxSize).toBe(longestText.length); 147 | expect(fdFn([87, longestText, '2019-01-01']).dataTypeName).toBe(DataTypeName.String); 148 | }); 149 | 150 | it('check value types array', () => { 151 | const fields = getFieldsInfo([1, 2, 3]); 152 | 153 | expect(fields.length).toBe(1); 154 | expect(fields[0].dataTypeName).toBe(DataTypeName.WholeNumber); 155 | expect(fields[0].fieldName).toBe('_value_'); 156 | 157 | expect(getFieldsInfo(['a', 'b'])[0].dataTypeName).toBe(DataTypeName.String); 158 | expect(getFieldsInfo(['a', 'b']).length).toBe(1); 159 | 160 | expect(getFieldsInfo(['2021-06-02', '2021-06-02']).length).toBe(1); 161 | expect(getFieldsInfo(['2021-06-02', '2021-06-02'])[0].dataTypeName).toBe(DataTypeName.Date); 162 | }); 163 | }); 164 | -------------------------------------------------------------------------------- /src/array/joins.ts: -------------------------------------------------------------------------------- 1 | import { Selector } from '../types'; 2 | import { fieldSelector } from '../_internals'; 3 | 4 | function verifyJoinArgs( 5 | leftArray: TLeft[], 6 | rightArray: TRight[], 7 | leftKeySelector: (item: TLeft) => string, 8 | rightKeySelector: (item: TRight) => string, 9 | resultSelector: (leftItem: TLeft | null, rightItem: TRight | null) => TResult 10 | ): void { 11 | if (!leftArray || !Array.isArray(leftArray)) { 12 | throw Error('leftArray is not provided or not a valid'); 13 | } 14 | if (!rightArray || !Array.isArray(rightArray)) { 15 | throw Error('rightArray is not provided or not a valid'); 16 | } 17 | 18 | if (typeof leftKeySelector !== 'function') { 19 | throw Error('leftKeySelector is not provided or not a valid function'); 20 | } 21 | 22 | if (typeof rightKeySelector !== 'function') { 23 | throw Error('rightKeySelector is not provided or not a valid function'); 24 | } 25 | 26 | if (typeof resultSelector !== 'function') { 27 | throw Error('resultSelector is not provided or not a valid function'); 28 | } 29 | } 30 | 31 | function leftOrInnerJoin( 32 | isInnerJoin: boolean, 33 | leftArray: TLeft[], 34 | rightArray: TRight[], 35 | leftKey: string | string[] | Selector, 36 | rightKey: string | string[] | Selector, 37 | resultSelector: (leftItem: TLeft | null, rightItem: TRight | null) => TResult 38 | ): TResult[] { 39 | const leftKeySelector = fieldSelector(leftKey); 40 | const rightKeySelector = fieldSelector(rightKey); 41 | 42 | verifyJoinArgs(leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); 43 | 44 | // build a lookup map 45 | const rightArrayMap: Record = Object.create(null); 46 | for (const item of rightArray) { 47 | rightArrayMap[rightKeySelector(item)] = item; 48 | } 49 | 50 | const result: TResult[] = []; 51 | for (const leftItem of leftArray) { 52 | const leftKey = leftKeySelector(leftItem); 53 | const rightItem = rightArrayMap[leftKey] || null; 54 | 55 | if (isInnerJoin && !rightItem) { 56 | continue; 57 | } 58 | 59 | const resultItem = resultSelector(leftItem, rightItem); 60 | 61 | // if result is null then probably a left item was modified 62 | result.push((resultItem || leftItem) as TResult); 63 | } 64 | 65 | return result; 66 | } 67 | 68 | /** 69 | * leftJoin returns all elements from the left array (leftArray), and the matched elements from the right array (rightArray). 70 | * The result is NULL from the right side, if there is no match. 71 | * @param leftArray array for left side in a join 72 | * @param rightArray array for right side in a join 73 | * @param leftKey A key from left side array. What can be as a fieldName, multiple fields or key Selector 74 | * @param rightKey A key from right side array. what can be as a fieldName, multiple fields or key Selector 75 | * @param resultSelector A callback function that returns result value 76 | */ 77 | export function leftJoin( 78 | leftArray: TLeft[], 79 | rightArray: TRight[], 80 | leftKeySelector: string | string[] | Selector, 81 | rightKeySelector: string | string[] | Selector, 82 | resultSelector: (leftItem: TLeft | null, rightItem: TRight | null) => TResult 83 | ): TResult[] { 84 | return leftOrInnerJoin( 85 | false, 86 | leftArray, 87 | rightArray, 88 | leftKeySelector, 89 | rightKeySelector, 90 | resultSelector 91 | ); 92 | } 93 | 94 | /** 95 | * innerJoin - Joins two arrays together by selecting elements that have matching values in both arrays. 96 | * If there are elements in any array that do not have matches in other array, these elements will not be shown! 97 | * @param leftArray array for left side in a join 98 | * @param rightArray array for right side in a join 99 | * @param leftKey A key from left side array. What can be as a fieldName, multiple fields or key Selector 100 | * @param rightKey A key from right side array. what can be as a fieldName, multiple fields or key Selector 101 | * @param resultSelector A callback function that returns result value 102 | */ 103 | export function innerJoin( 104 | leftArray: TLeft[], 105 | rightArray: TRight[], 106 | leftKey: string | string[] | Selector, 107 | rightKey: string | string[] | Selector, 108 | resultSelector: (leftItem: TLeft | null, rightItem: TRight | null) => TResult 109 | ): TResult[] { 110 | return leftOrInnerJoin(true, leftArray, rightArray, leftKey, rightKey, resultSelector); 111 | } 112 | 113 | /** 114 | * fullJoin returns all elements from the left array (leftArray), and all elements from the right array (rightArray). 115 | * The result is NULL from the right/left side, if there is no match. 116 | * @param leftArray array for left side in a join 117 | * @param rightArray array for right side in a join 118 | * @param leftKey A key from left side array. What can be as a fieldName, multiple fields or key Selector 119 | * @param rightKey A key from right side array. what can be as a fieldName, multiple fields or key Selector 120 | * @param resultSelector A callback function that returns result value 121 | */ 122 | export function fullJoin( 123 | leftArray: TLeft[], 124 | rightArray: TRight[], 125 | leftKey: string | string[] | Selector, 126 | rightKey: string | string[] | Selector, 127 | resultSelector: (leftItem: TLeft | null, rightItem: TRight | null) => TResult 128 | ): TResult[] { 129 | const leftKeySelector = fieldSelector(leftKey); 130 | const rightKeySelector = fieldSelector(rightKey); 131 | 132 | verifyJoinArgs(leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); 133 | 134 | // build a lookup maps for both arrays. 135 | // so, both of them have to be unique, otherwise it will flattern result 136 | const leftArrayMap: Record = Object.create(null); 137 | for (const item of leftArray) { 138 | leftArrayMap[leftKeySelector(item)] = item; 139 | } 140 | 141 | const rightArrayMap: Record = Object.create(null); 142 | for (const item of rightArray) { 143 | rightArrayMap[rightKeySelector(item)] = item; 144 | } 145 | 146 | const result: TResult[] = []; 147 | for (const leftItem of leftArray) { 148 | const leftKey = leftKeySelector(leftItem); 149 | const rightItem = rightArrayMap[leftKey] || null; 150 | 151 | const resultItem = resultSelector(leftItem, rightItem); 152 | 153 | // if result is null then probably a left item was modified 154 | result.push((resultItem || leftItem) as TResult); 155 | if (rightItem) { 156 | delete rightArrayMap[leftKey]; 157 | } 158 | } 159 | 160 | // add remaining right items 161 | for (const rightItemKey in rightArrayMap) { 162 | const rightItem = rightArrayMap[rightItemKey]; 163 | const resultItem = resultSelector(null, rightItem); 164 | 165 | // if result is null then probably a left item was modified 166 | result.push(resultItem || (rightItem as unknown as TResult)); 167 | } 168 | 169 | return result; 170 | } 171 | 172 | /** 173 | * merges elements from two arrays. It appends source element or overrides to target array based on matching keys provided 174 | * @param targetArray target array 175 | * @param sourceArray source array 176 | * @param targetKey tartget key field, arry of fields or field serlector 177 | * @param sourceKey source key field, arry of fields or field serlector 178 | */ 179 | export function merge( 180 | targetArray: TTarget[], 181 | sourceArray: TSource[], 182 | targetKey: string | string[] | Selector, 183 | sourceKey: string | string[] | Selector 184 | ): TTarget[] { 185 | const targetKeySelector = fieldSelector(targetKey); 186 | const sourceKeySelector = fieldSelector(sourceKey); 187 | verifyJoinArgs(targetArray, sourceArray, targetKeySelector, sourceKeySelector, () => false); 188 | 189 | // build a lookup maps for both arrays. 190 | // so, both of them have to be unique, otherwise it will flattern result 191 | const targetArrayMap: Record = Object.create(null); 192 | for (const item of targetArray) { 193 | targetArrayMap[targetKeySelector(item)] = item; 194 | } 195 | 196 | const sourceArrayMap: Record = Object.create(null); 197 | for (const item of sourceArray) { 198 | sourceArrayMap[sourceKeySelector(item)] = item; 199 | } 200 | 201 | for (const sourceItemKey of Object.keys(sourceArrayMap)) { 202 | const sourceItem = sourceArrayMap[sourceItemKey]; 203 | if (!targetArrayMap[sourceItemKey]) { 204 | targetArray.push(sourceItem as unknown as TTarget); 205 | } else { 206 | // merge properties in 207 | Object.assign(targetArrayMap[sourceItemKey], sourceItem); 208 | } 209 | } 210 | 211 | return targetArray; 212 | } 213 | -------------------------------------------------------------------------------- /src/array/transform.ts: -------------------------------------------------------------------------------- 1 | import { Selector, Predicate } from '../types'; 2 | import { fieldSelector } from '../_internals'; 3 | import { sum } from './stats'; 4 | 5 | /** 6 | * Groups array items based on elementSelector function 7 | * @param array The array to process. 8 | * @param elementSelector Function invoked per iteration. 9 | */ 10 | export function groupBy(array: T[], groupByFields: string | string[] | Selector): T[][] { 11 | if (!Array.isArray(array)) { 12 | throw Error('An array is not provided'); 13 | } 14 | 15 | if (!array.length) { 16 | return array as unknown as T[][]; 17 | } 18 | 19 | const groups: { [key: string]: T[] } = {}; 20 | 21 | const elementSelector = fieldSelector(groupByFields as Selector); 22 | 23 | for (let i = 0; i < array.length; i++) { 24 | const item = array[i]; 25 | const group = elementSelector(item); 26 | groups[group] = groups[group] || []; 27 | groups[group].push(item); 28 | } 29 | 30 | return Object.values(groups); 31 | } 32 | 33 | /** 34 | * Returns a distinct elements from array. 35 | * Optional parameter *elementSelector* will create new array based on a callback function, 36 | * then will eliminate dublicates 37 | * @param array 38 | * @param elementSelector 39 | * @returns 40 | */ 41 | export function distinct(array: T[], elementSelector?: Selector): (T | V)[] { 42 | if (elementSelector) { 43 | const mapped = array.map(elementSelector); 44 | return Array.from(new Set(mapped)); 45 | } 46 | return Array.from(new Set(array)); 47 | } 48 | /** 49 | * Flatten Object or array of objects 50 | * Object 51 | * [{ a: 1, d:{d1: 22, d2: 33} }, { b: 2, d:{d1:221, d2:331} }] 52 | * will become 53 | * [{ a: 1, d.d1: 22, d.d2: 33 }, { b: 2, d.d1: 221, d.d2: 331 }] 54 | * @param data 55 | * @returns 56 | */ 57 | 58 | export function unflattenObject(data: unknown): unknown { 59 | 60 | 61 | function setProperty(data: Record, pName: string, value: unknown): void { 62 | const pNames = pName.split('.') 63 | let parent: Record = data 64 | 65 | for (let i = 0; i < pNames.length - 1; i++) { 66 | if (pNames[i] in parent) { 67 | parent = parent[pNames[i]] as Record 68 | } else { 69 | parent[pNames[i]] = {} 70 | parent = parent[pNames[i]] as Record 71 | } 72 | } 73 | parent[pNames[pNames.length - 1]] = value 74 | } 75 | 76 | if (Array.isArray(data)) { 77 | const arr = []; 78 | for (let i = 0; i < data.length; i++) { 79 | const obj = unflattenObject(data[i]); 80 | arr.push(obj); 81 | } 82 | return arr; 83 | } else { 84 | const obj: Record = {} 85 | const keys = Object.keys(data as Record); 86 | for (let i = 0; i < keys.length; i++) { 87 | setProperty(obj, keys[i], (data as Record)[keys[i]]) 88 | } 89 | return obj 90 | } 91 | } 92 | 93 | export function flattenObject(data: unknown): Record[] { 94 | if (!data || typeof data !== 'object') { 95 | return data as Record[]; 96 | } 97 | 98 | function iterate(rootData: Record, obj: Record, prefix: string): void { 99 | const keys = Object.keys(rootData); 100 | 101 | for (let i = 0; i < keys.length; i++) { 102 | const pName = prefix ? `${prefix}.${keys[i]}` : keys[i]; 103 | 104 | const value = rootData[keys[i]]; 105 | if ( 106 | typeof value === 'object' && 107 | value !== null && 108 | value !== undefined && 109 | !(value instanceof Date) 110 | ) { 111 | iterate(value as Record, obj, pName); 112 | } else if (typeof value === 'function') { 113 | continue; 114 | } else { 115 | obj[pName] = value; 116 | } 117 | } 118 | } 119 | 120 | if (Array.isArray(data)) { 121 | const arr = []; 122 | for (let i = 0; i < data.length; i++) { 123 | const obj = flattenObject(data[i]); 124 | arr.push(obj); 125 | } 126 | return arr as unknown as Record[]; 127 | } else { 128 | const obj: Record = {}; 129 | 130 | iterate(data as Record, obj, ''); 131 | 132 | return obj as unknown as Record[]; 133 | } 134 | } 135 | 136 | /** 137 | * Flattens array. 138 | * @param array The array to flatten recursively. 139 | * @example 140 | * flatten([1, 4, [2, [5, 5, [9, 7]], 11], 0]); // length 9 141 | */ 142 | export function flatten(array: T[]): T[] { 143 | if (!Array.isArray(array)) { 144 | throw Error('An array is not provided'); 145 | } 146 | 147 | if (!array.length) { 148 | return array; 149 | } 150 | 151 | let res: T[] = []; 152 | const length = array.length; 153 | 154 | for (let i = 0; i < length; i++) { 155 | const value = array[i]; 156 | if (Array.isArray(value)) { 157 | res = [...res, ...flatten(value as T[])]; 158 | } else { 159 | res.push(value); 160 | } 161 | } 162 | return res; 163 | } 164 | 165 | /** 166 | * Returns a reshaped array based on unique column values. 167 | * @param array array to pivot 168 | * @param rowFields row fields (or index fields). It can be one or more field names 169 | * @param columnField a field which values will be used to create columns 170 | * @param dataField a dataField which will be aggrated with aggregate function and groupped by rows and columns 171 | * @param aggFunction an aggregation function. Default value is sum. data field will be aggregated by this function 172 | * @param columnValues an optional initial column values. Use it to define a set of columns/values you would expect 173 | */ 174 | export function pivot>( 175 | array: T[], 176 | rowFields: string | string[], 177 | columnField: string, 178 | dataField: string, 179 | aggFunction?: (array: unknown[]) => unknown | null, 180 | columnValues?: string[] 181 | ): Array> { 182 | if (!Array.isArray(array)) { 183 | throw Error('An array is not provided'); 184 | } 185 | 186 | if (!array.length) { 187 | return array; 188 | } 189 | 190 | const groups: { [key: string]: T[] } = Object.create(null); 191 | columnValues = columnValues || []; 192 | aggFunction = aggFunction || ((a: unknown[]): number | null => sum(a as number[])); 193 | 194 | const elementSelector = fieldSelector(rowFields); 195 | 196 | rowFields = Array.isArray(rowFields) ? rowFields : [rowFields]; 197 | 198 | // group by rows 199 | for (let i = 0; i < array.length; i++) { 200 | const item = array[i]; 201 | const group = elementSelector(item); 202 | groups[group] = groups[group] || []; 203 | groups[group].push(item); 204 | 205 | // accumulate column values 206 | const columnValue = item[columnField] as string; 207 | if (columnValues.indexOf(columnValue) < 0) { 208 | columnValues.push(columnValue); 209 | } 210 | } 211 | 212 | const result: Array> = []; 213 | for (const groupName of Object.keys(groups)) { 214 | const item: Record = Object.create(null); 215 | // row fields first 216 | for (const rowField of rowFields) { 217 | item[rowField] = groups[groupName][0][rowField]; 218 | } 219 | 220 | // then aggregated data for each colum value 221 | for (const columnValue of columnValues) { 222 | const dataArray = groups[groupName] 223 | .filter(r => r[columnField] === columnValue) 224 | .map(r => r[dataField]); 225 | item[columnValue] = aggFunction(dataArray); 226 | } 227 | 228 | result.push(item); 229 | } 230 | 231 | return result; 232 | } 233 | 234 | /** 235 | * Transpose rows to columns in an array 236 | * @param data 237 | */ 238 | export function transpose>(data: T[]): Array> { 239 | if (!Array.isArray(data)) { 240 | throw Error('An array is not provided'); 241 | } 242 | 243 | if (!data.length) { 244 | return data; 245 | } 246 | 247 | return Object.keys(data[0]).map(key => { 248 | const res: Record = {}; 249 | data.forEach((item, i) => { 250 | if (i === 0) { 251 | res.fieldName = key; 252 | } 253 | 254 | res['row' + i] = item[key]; 255 | }); 256 | return res; 257 | }); 258 | } 259 | 260 | /** 261 | * Creates new array based on selector. 262 | * @param array The array to process. 263 | * @param elementSelector Function invoked per iteration. 264 | */ 265 | export function select(data: T[], selector: string | string[] | Selector): string[] { 266 | if (!Array.isArray(data)) { 267 | throw Error('An array is not provided'); 268 | } 269 | 270 | if (!data.length) { 271 | return []; 272 | } 273 | const elementSelector = fieldSelector(selector); 274 | return data.map(elementSelector); 275 | } 276 | 277 | /** 278 | * Filters array based on predicate function. 279 | * @param array The array to process. 280 | * @param elementSelector Function invoked per iteration. 281 | */ 282 | export function where(data: T[], predicate: Predicate): T[] { 283 | if (!Array.isArray(data)) { 284 | throw Error('An array is not provided'); 285 | } 286 | 287 | return data.filter(predicate); 288 | } 289 | -------------------------------------------------------------------------------- /src/array/stats.ts: -------------------------------------------------------------------------------- 1 | import { Selector, Predicate, ScalarType } from '../types'; 2 | import { parseNumber } from '../utils'; 3 | import { isArrayEmptyOrNull } from './utils'; 4 | 5 | function fieldSelector(field?: string | Selector): Selector { 6 | if (!field) { 7 | return (item: T): ScalarType => item as unknown as ScalarType; 8 | } 9 | return typeof field === 'function' 10 | ? (field as Selector) 11 | : (item: T): ScalarType => (item as Record)[String(field)] as ScalarType; 12 | } 13 | 14 | function fieldComparator(field?: string | Selector): (a: T, b: T) => number { 15 | return (a: T, b: T): number => { 16 | const selector = fieldSelector(field); 17 | const aVal = parseNumber(selector(a) as ScalarType); 18 | const bVal = parseNumber(selector(b) as ScalarType); 19 | 20 | if (bVal === undefined) { 21 | return 1; 22 | } 23 | 24 | if (aVal === undefined) { 25 | return -1; 26 | } 27 | 28 | return aVal - bVal >= 0 ? 1 : -1; 29 | }; 30 | } 31 | 32 | function getNumberValuesArray(array: T[], field?: string | Selector): number[] { 33 | const elementSelector = fieldSelector(field); 34 | return array 35 | .map(item => parseNumber(elementSelector(item) as ScalarType)) 36 | .filter(v => v !== undefined) as number[]; 37 | } 38 | 39 | /** 40 | * Sum of items in array. 41 | * @param array The array to process. 42 | * @param elementSelector Function invoked per iteration. 43 | * @returns Sum of array. 44 | * @public 45 | * @example 46 | * sum([1, 2, 5]); // 8 47 | * 48 | * sum([{ val: 1 }, { val: 5 }], i => i.val); // 6 49 | */ 50 | export function sum(array: T[], field?: Selector | string): number | null { 51 | if (isArrayEmptyOrNull(array)) { 52 | return null; 53 | } 54 | 55 | const elementSelector = fieldSelector(field); 56 | 57 | let sum = 0; 58 | for (const item of array) { 59 | const numberVal = parseNumber(elementSelector(item) as ScalarType); 60 | if (numberVal) { 61 | sum += numberVal; 62 | } 63 | } 64 | 65 | return sum; 66 | } 67 | 68 | /** 69 | * Average of array items. 70 | * @param array The array to process. 71 | * @param elementSelector Function invoked per iteration. 72 | * @public 73 | * @example 74 | * avg([1, 5, 3]); // 3 75 | */ 76 | export function avg(array: T[], field?: Selector | string): number | null { 77 | if (isArrayEmptyOrNull(array)) { 78 | return null; 79 | } 80 | 81 | const elementSelector = fieldSelector(field); 82 | 83 | const s = sum(array, elementSelector); 84 | 85 | return s ? s / array.length : null; 86 | } 87 | 88 | /** 89 | * Computes the minimum value of array. 90 | * @param array The array to process. 91 | * @param elementSelector Function invoked per iteration. 92 | */ 93 | export function min(array: T[], field?: Selector | string): number | Date | null { 94 | const elementSelector = fieldSelector(field); 95 | const numberArray = getNumberValuesArray(array, elementSelector); 96 | if (isArrayEmptyOrNull(numberArray)) { 97 | return null; 98 | } 99 | const min = Math.min(...numberArray); 100 | const item = field ? elementSelector(array[0]) : array[0]; 101 | if (item instanceof Date) { 102 | return new Date(min); 103 | } 104 | return min; 105 | } 106 | 107 | /** 108 | * Computes the maximum value of array. 109 | * @public 110 | * @param array The array to process. 111 | * @param elementSelector Function invoked per iteration. 112 | */ 113 | export function max(array: T[], field?: Selector | string): number | Date | null { 114 | const elementSelector = fieldSelector(field); 115 | const numberArray = getNumberValuesArray(array, elementSelector); 116 | if (isArrayEmptyOrNull(numberArray)) { 117 | return null; 118 | } 119 | const max = Math.max(...numberArray); 120 | const item = field ? elementSelector(array[0]) : array[0]; 121 | if (item instanceof Date) { 122 | return new Date(max); 123 | } 124 | return max; 125 | } 126 | 127 | /** 128 | * Count of elements in array. 129 | * @public 130 | * @param array The array to process. 131 | * @param predicate Predicate function invoked per iteration. 132 | */ 133 | export function count(array: T[], predicate?: Predicate): number | null { 134 | if (!array || !Array.isArray(array)) return null; 135 | 136 | if (!predicate || typeof predicate !== 'function') { 137 | return array.length; 138 | } 139 | return array.filter(predicate).length; 140 | } 141 | 142 | /** 143 | * Gets first item in array satisfies predicate. 144 | * @param array The array to process. 145 | * @param predicate Predicate function invoked per iteration. 146 | */ 147 | export function first(array: T[], predicate?: Predicate): T | null { 148 | if (isArrayEmptyOrNull(array)) { 149 | return null; 150 | } 151 | 152 | if (!predicate) { 153 | return array[0]; 154 | } 155 | for (let i = 0; i < array.length; i++) { 156 | if (predicate(array[i])) { 157 | return array[i]; 158 | } 159 | } 160 | return null; 161 | } 162 | 163 | /** 164 | * Gets last item in array satisfies predicate. 165 | * @param array The array to process. 166 | * @param predicate Predicate function invoked per iteration. 167 | */ 168 | export function last(array: T[], predicate?: Predicate): T | null { 169 | if (isArrayEmptyOrNull(array)) { 170 | return null; 171 | } 172 | 173 | let lastIndex = array.length - 1; 174 | if (!predicate) { 175 | return array[lastIndex]; 176 | } 177 | 178 | for (; lastIndex >= 0; lastIndex--) { 179 | if (predicate(array[lastIndex])) { 180 | return array[lastIndex]; 181 | } 182 | } 183 | 184 | return null; 185 | } 186 | 187 | /** 188 | * Gets counts map of values returned by `elementSelector`. 189 | * @param array The array to process. 190 | * @param elementSelector Function invoked per iteration. 191 | */ 192 | export function countBy(array: T[], elementSelector: Selector): Record { 193 | if (!array || !Array.isArray(array)) { 194 | throw Error('No array provided'); 195 | } 196 | 197 | const results: Record = {}; 198 | const length = array.length; 199 | 200 | for (let i = 0; i < length; i++) { 201 | const item = array[i]; 202 | const group = String(elementSelector(item)); 203 | results[group] = results[group] || 0; 204 | results[group]++; 205 | } 206 | 207 | return results; 208 | } 209 | 210 | /** 211 | * Get mean of an array. 212 | * @param array The array to process. 213 | * @param field Property name or Selector function invoked per iteration. 214 | */ 215 | export function mean(array: T[], field?: Selector | string): number | null { 216 | if (isArrayEmptyOrNull(array)) { 217 | return null; 218 | } 219 | 220 | let res = 0; 221 | for (let i = 0, c = 0, len = array.length; i < len; ++i) { 222 | const selector = fieldSelector(field); 223 | const val = parseNumber(selector(array[i]) as ScalarType); 224 | if (typeof val === 'number') { 225 | const delta = val - res; 226 | res = res + delta / ++c; 227 | } 228 | } 229 | return res; 230 | } 231 | 232 | /** 233 | * Get quantile of a sorted array. 234 | * @param array The array to process. 235 | * @param field Property name or Selector function invoked per iteration. 236 | * @param p quantile. 237 | */ 238 | export function quantile(array: T[], p: number, field?: Selector | string): number | null { 239 | if (isArrayEmptyOrNull(array)) { 240 | return null; 241 | } 242 | 243 | const numberArray = getNumberValuesArray(array, field); 244 | const len = (numberArray.length - 1) * p + 1; 245 | const l = Math.floor(len); 246 | const val = numberArray[l - 1]; 247 | const e = len - l; 248 | return e ? val + e * (numberArray[l] - val) : val; 249 | } 250 | 251 | /** 252 | * Get sample variance of an array. 253 | * @param array The array to process. 254 | * @param field Property name or Selector function invoked per iteration. 255 | */ 256 | export function variance(array: T[], field?: Selector | string): number | null { 257 | if (isArrayEmptyOrNull(array)) { 258 | return null; 259 | } 260 | 261 | const elementSelector = fieldSelector(field); 262 | if (!Array.isArray(array) || array.length < 2) { 263 | return 0; 264 | } 265 | let mean = 0, 266 | M2 = 0, 267 | c = 0; 268 | for (let i = 0; i < array.length; ++i) { 269 | const val = parseNumber(elementSelector(array[i]) as ScalarType); 270 | if (typeof val === 'number') { 271 | const delta = val - mean; 272 | mean = mean + delta / ++c; 273 | M2 = M2 + delta * (val - mean); 274 | } 275 | } 276 | M2 = M2 / (c - 1); 277 | return M2; 278 | } 279 | 280 | /** 281 | * Get the sample standard deviation of an array. 282 | * @param array The array to process. 283 | * @param field Property name or Selector function invoked per iteration. 284 | */ 285 | export function stdev(array: T[], field?: Selector | string): number | null { 286 | if (isArrayEmptyOrNull(array)) { 287 | return null; 288 | } 289 | 290 | const varr = variance(array, field); 291 | 292 | if (varr === null) { 293 | return null; 294 | } 295 | 296 | return Math.sqrt(varr); 297 | } 298 | 299 | /** 300 | * Get median of an array. 301 | * @param array The array to process. 302 | * @param field Property name or Selector function invoked per iteration. 303 | */ 304 | export function median(array: T[], field?: Selector | string): number | null { 305 | if (isArrayEmptyOrNull(array)) { 306 | return null; 307 | } 308 | 309 | array.sort(fieldComparator(field)); 310 | return quantile(getNumberValuesArray(array, field), 0.5); 311 | } 312 | -------------------------------------------------------------------------------- /src/utils/dsv-parser.ts: -------------------------------------------------------------------------------- 1 | import { 2 | parseNumberOrNull, 3 | parseDatetimeOrNull, 4 | workoutDataType, 5 | parseBooleanOrNull, 6 | dateToString 7 | } from './helpers'; 8 | import { 9 | ParsingOptions, 10 | ScalarType, 11 | ScalarObject, 12 | StringsDataTable, 13 | FieldDescription, 14 | DataTypeName 15 | } from '../types'; 16 | 17 | type ParsingContext = { 18 | content: string; 19 | currentIndex: number; 20 | }; 21 | 22 | const EmptySymbol = '_#EMPTY#_'; 23 | 24 | function getObjectElement( 25 | fieldDescs: FieldDescription[], 26 | tokens: string[], 27 | options: ParsingOptions 28 | ): ScalarObject { 29 | const obj = Object.create(null); 30 | for (let i = 0; i < fieldDescs.length; i++) { 31 | const fieldDesc = fieldDescs[i]; 32 | const fieldName = fieldDesc.fieldName; 33 | let value: ScalarType = tokens[i]; 34 | const dateFields = options?.dateFields?.map(r => (Array.isArray(r) ? r[0] : r)) || []; 35 | 36 | if (options.textFields && options.textFields?.indexOf(fieldName) >= 0) { 37 | value = tokens[i]; 38 | } else if ( 39 | fieldDesc.dataTypeName === DataTypeName.DateTime || 40 | fieldDesc.dataTypeName === DataTypeName.Date || 41 | dateFields.indexOf(fieldName) >= 0 42 | ) { 43 | const ind = dateFields.indexOf(fieldName); 44 | const dtField = ind >= 0 ? options?.dateFields[ind] : []; 45 | const format = Array.isArray(dtField) ? dtField[1] : null; 46 | value = parseDatetimeOrNull(value as string, format || null); 47 | } else if ( 48 | fieldDesc.dataTypeName === DataTypeName.WholeNumber || 49 | fieldDesc.dataTypeName === DataTypeName.FloatNumber || 50 | fieldDesc.dataTypeName === DataTypeName.BigIntNumber || 51 | (options.numberFields && options.numberFields.indexOf(fieldName) >= 0) 52 | ) { 53 | value = parseNumberOrNull(value as string); 54 | } else if ( 55 | fieldDesc.dataTypeName === DataTypeName.Boolean || 56 | (options.booleanFields && options.booleanFields.indexOf(fieldName) >= 0) 57 | ) { 58 | value = parseBooleanOrNull(value as string); 59 | } 60 | 61 | obj[fieldName] = value === EmptySymbol ? '' : value; 62 | } 63 | return obj; 64 | } 65 | 66 | function nextLineTokens(context: ParsingContext, delimiter = ','): string[] { 67 | const tokens: string[] = []; 68 | let token = ''; 69 | 70 | function elementAtOrNull(arr: string, index: number): string | null { 71 | return arr.length > index ? arr[index] : null; 72 | } 73 | 74 | do { 75 | const currentChar = context.content[context.currentIndex]; 76 | if (currentChar === '\r') { 77 | continue; 78 | } 79 | 80 | if (currentChar === '\n') { 81 | if (context.content[context.currentIndex + 1] === '\r') { 82 | context.currentIndex++; 83 | } 84 | break; 85 | } 86 | 87 | if (token.length === 0 && currentChar === '"') { 88 | if ( 89 | elementAtOrNull(context.content, context.currentIndex + 1) === '"' && 90 | elementAtOrNull(context.content, context.currentIndex + 2) !== '"' 91 | ) { 92 | // just empty string 93 | token = EmptySymbol; 94 | context.currentIndex++; 95 | } else { 96 | // enumerate till the end of quote 97 | while (context.content[++context.currentIndex] !== '"') { 98 | token += context.content[context.currentIndex]; 99 | 100 | // check if we need to escape "" 101 | if ( 102 | elementAtOrNull(context.content, context.currentIndex + 1) === '"' && 103 | elementAtOrNull(context.content, context.currentIndex + 2) === '"' 104 | ) { 105 | token += '"'; 106 | context.currentIndex += 2; 107 | } 108 | } 109 | } 110 | } else if (currentChar === delimiter) { 111 | tokens.push(token); 112 | token = ''; 113 | } else { 114 | token += currentChar; 115 | } 116 | } while (++context.currentIndex < context.content.length); 117 | 118 | tokens.push(token); 119 | return tokens; 120 | } 121 | 122 | function parseLineTokens(content: string, options: ParsingOptions): StringsDataTable { 123 | const ctx = { 124 | content: content, 125 | currentIndex: 0 126 | } as ParsingContext; 127 | content = content || ''; 128 | const delimiter = options.delimiter || ','; 129 | 130 | const result = { 131 | fieldDescriptions: [] as FieldDescription[], 132 | rows: [] as ScalarType[][] 133 | } as StringsDataTable; 134 | let lineNumber = 0; 135 | let fieldNames: string[] | null = null; 136 | const uniqueValues: string[][] = []; 137 | 138 | do { 139 | const rowTokens = nextLineTokens(ctx, delimiter); 140 | 141 | // skip if all tokens are empty 142 | if (rowTokens.filter(f => !f || !f.length).length === rowTokens.length) { 143 | lineNumber++; 144 | continue; 145 | } 146 | 147 | // skip rows based skipRows value 148 | if (lineNumber < options.skipRows) { 149 | lineNumber++; 150 | continue; 151 | } 152 | 153 | // skip rows based on skipUntil call back 154 | if (!fieldNames && typeof options.skipUntil === 'function' && !options.skipUntil(rowTokens)) { 155 | lineNumber++; 156 | continue; 157 | } 158 | 159 | if (!fieldNames) { 160 | // fieldName is used as indicator on whether data rows handling started 161 | fieldNames = []; 162 | const fieldDescriptions = []; 163 | 164 | for (let i = 0; i < rowTokens.length; i++) { 165 | // if empty then _ 166 | let token = rowTokens[i].trim().length ? rowTokens[i].trim() : '_'; 167 | 168 | if (!options.keepOriginalHeaders) { 169 | token = token.replace(/\W/g, '_'); 170 | } 171 | 172 | // just to ensure no dublicated field names 173 | fieldNames.push(fieldNames.indexOf(token) >= 0 ? `${token}_${i}` : token); 174 | 175 | fieldDescriptions.push({ 176 | fieldName: fieldNames[fieldNames.length - 1], 177 | isNullable: false, 178 | isUnique: true, 179 | index: i 180 | } as FieldDescription); 181 | 182 | uniqueValues.push([]); 183 | } 184 | 185 | result.fieldDescriptions = fieldDescriptions; 186 | result.fieldNames = fieldNames; 187 | 188 | lineNumber++; 189 | continue; 190 | } 191 | 192 | if (typeof options.takeWhile === 'function' && fieldNames && !options.takeWhile(rowTokens)) { 193 | break; 194 | } 195 | const rowValues: string[] = []; 196 | 197 | // analyze each cell in a row 198 | for (let i = 0; i < Math.min(rowTokens.length, result.fieldDescriptions.length); i++) { 199 | const fDesc = result.fieldDescriptions[i]; 200 | let value: string | null = rowTokens[i]; 201 | 202 | if (value === null || value === undefined || value.length === 0) { 203 | fDesc.isNullable = true; 204 | } else if (value !== EmptySymbol) { 205 | const newType = workoutDataType(value, fDesc.dataTypeName); 206 | if (newType !== fDesc.dataTypeName) { 207 | fDesc.dataTypeName = newType; 208 | } 209 | 210 | if ( 211 | (fDesc.dataTypeName == DataTypeName.String || 212 | fDesc.dataTypeName == DataTypeName.LargeString) && 213 | String(value).length > (fDesc.maxSize || 0) 214 | ) { 215 | fDesc.maxSize = String(value).length; 216 | } 217 | } 218 | 219 | if (fDesc.isUnique) { 220 | if (uniqueValues[i].indexOf(value) >= 0) { 221 | fDesc.isUnique = false; 222 | } else { 223 | uniqueValues[i].push(value); 224 | } 225 | } 226 | 227 | if (value === EmptySymbol) { 228 | value = 229 | fDesc.dataTypeName === DataTypeName.String || 230 | fDesc.dataTypeName === DataTypeName.LargeString 231 | ? '' 232 | : null; 233 | } else if (!value.length) { 234 | value = null; 235 | } 236 | rowValues.push(value as string); 237 | } 238 | 239 | // no need for null or empty objects 240 | result.rows.push(rowValues); 241 | lineNumber++; 242 | } while (++ctx.currentIndex < ctx.content.length); 243 | 244 | result.fieldDataTypes = result.fieldDescriptions.map(f => f.dataTypeName as DataTypeName); 245 | return result; 246 | } 247 | 248 | export function parseCsv(content: string, options?: ParsingOptions): ScalarObject[] { 249 | content = content || ''; 250 | options = Object.assign(new ParsingOptions(), options || {}); 251 | if (!content.length) { 252 | return []; 253 | } 254 | 255 | const table = parseLineTokens(content, options || new ParsingOptions()); 256 | 257 | const result: ScalarObject[] = []; 258 | for (let i = 0; i < table.rows.length; i++) { 259 | const obj = 260 | typeof options.elementSelector === 'function' 261 | ? options.elementSelector(table.fieldDescriptions, table.rows[i] as string[]) 262 | : getObjectElement(table.fieldDescriptions, table.rows[i] as string[], options); 263 | 264 | if (obj) { 265 | // no need for null or empty objects 266 | result.push(obj as ScalarObject); 267 | } 268 | } 269 | 270 | return result; 271 | } 272 | 273 | export function parseCsvToTable(content: string, options?: ParsingOptions): StringsDataTable { 274 | content = content || ''; 275 | 276 | if (!content.length) { 277 | return {} as StringsDataTable; 278 | } 279 | 280 | return parseLineTokens(content, options || new ParsingOptions()); 281 | } 282 | 283 | export function toCsv(array: ScalarObject[], delimiter = ','): string { 284 | array = array || []; 285 | 286 | const headers: string[] = []; 287 | 288 | // workout all headers 289 | for (const item of array) { 290 | for (const name in item) { 291 | if (headers.indexOf(name) < 0) { 292 | headers.push(name); 293 | } 294 | } 295 | } 296 | 297 | // create a csv string 298 | const lines = array.map(item => { 299 | const values: string[] = []; 300 | for (const name of headers) { 301 | let value: ScalarType = item[name]; 302 | if (value instanceof Date) { 303 | value = dateToString(value); 304 | } else if (value && typeof value === 'object') { 305 | value = `"${JSON.stringify(value).replace(new RegExp('"', 'g'), '""')}"`; 306 | } else if ( 307 | typeof value === 'string' && 308 | (value.indexOf(delimiter) >= 0 || value.indexOf('"') >= 0) 309 | ) { 310 | // excel style csv 311 | value = value.replace(new RegExp('"', 'g'), '""'); 312 | value = `"${value}"`; 313 | } 314 | value = value !== null && value !== undefined ? value : ''; 315 | values.push(String(value)); 316 | } 317 | return values.join(delimiter); 318 | }); 319 | lines.unshift(headers.join(delimiter)); 320 | 321 | return lines.join('\n'); 322 | } 323 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dataPipe 2 | 3 | dataPipe is a data processing and data analytics library for JavaScript. Inspired by LINQ (C#) and Pandas (Python). It provides a facilities for data loading, data transformation, data analysis and other helpful data manipulation functions. 4 | 5 | Originally DataPipe project was created to power [JSPython](https://github.com/jspython-dev/jspython) and [Worksheet Systems](https://worksheet.systems) related projects, but it is also can be used as a standalone library for your data-driven JavaScript or JSPython applications on both the client (web browser) and server (NodeJS). 6 | 7 | ## Get started 8 | 9 | A quick way to use it in html 10 | 11 | ``` 12 | 13 | ``` 14 | 15 | or npm 16 | 17 | ``` 18 | npm install datapipe-js 19 | ``` 20 | 21 | ## A quick example 22 | 23 | JavaScript / TypeScript 24 | 25 | [StackBlitz example](https://stackblitz.com/edit/datapipe-js-examples?file=index.js) 26 | 27 | ```js 28 | const { dataPipe, avg, first } = require('datapipe-js'); 29 | const fetch = require('node-fetch'); 30 | 31 | async function main() { 32 | 33 | const dataUrl = "https://raw.githubusercontent.com/FalconSoft/sample-data/master/CSV/sample-testing-data-100.csv"; 34 | const csv = await (await fetch(dataUrl)).text(); 35 | 36 | return dataPipe() 37 | .fromCsv(csv) 38 | .groupBy(r => r.Country) 39 | .select(g => ({ 40 | country: first(g).Country, 41 | sales: dataPipe(g).sum(i => i.Sales), 42 | averageSales: avg(g, i => i.Sales), 43 | count: g.length 44 | }) 45 | ) 46 | .where(r => r.sales > 5000) 47 | .sort("sales DESC") 48 | .toArray(); 49 | } 50 | 51 | main() 52 | .then(console.log) 53 | .catch(console.error) 54 | ``` 55 | 56 | ## Data management functions 57 | 58 | All utlity functions can be used as a chaining (pipe) methods as well as a separately. In an example you will notice that to sum up `sales` we created a new dataPipe, but for an `averageSales` we used just a utility method `avg`. 59 | 60 | ### Data Loading 61 | 62 | Loading and parsing data from a common file formats like: CSV, JSON, TSV either from local variable 63 | 64 | - [**dataPipe**](https://www.datapipe-js.com/docs/datapipe#datapipe) (array) - accepts a JavaScript array 65 | 66 | - [**fromTable**](https://www.datapipe-js.com/docs/datapipe-js-utils#fromtable) (table) - converts a rows and columns list into array of sclalar objects 67 | 68 | - [**parseCsv**](https://www.datapipe-js.com/docs/datapipe-js-utils#parsecsv) (csvContent[, options]) - it loads a string content and process each row with optional but robust configuration options and callbacks e.g. skipRows, skipUntil, takeWhile, rowSelector, rowPredicate etc. This will automatically convert all types to numbers, datetimes or booleans if otherwise is not specified 69 | 70 | ### Data Transformation 71 | 72 | - [**select**](https://www.datapipe-js.com/docs/datapipe-js-array#select) (elementSelector) synonym **map** - creates a new element for each element in a pipe based on elementSelector callback. 73 | - [**where**](https://www.datapipe-js.com/docs/datapipe-js-array#where) (predicate) / **filter** - filters elements in a pipe based on predicate 74 | - [**groupBy**](https://www.datapipe-js.com/docs/datapipe-js-array#groupby) (keySelector) - groups elements in a pipe according to the keySelector callback-function. Returns a pipe with new group objects. 75 | - [**distinct**](https://www.datapipe-js.com/docs/datapipe-js-array#distinct) (elementSelector) / **unique** - returns distinct elements from array. Optional parameter *elementSelector* will create new array based on a callback function, then will eliminate dublicates 76 | - [**pivot**](https://www.datapipe-js.com/docs/datapipe-js-array#pivot) (array, rowFields, columnField, dataField, aggFunction?, columnValues?) - Returns a reshaped (pivoted) array based on unique column values. 77 | - [**transpose**](https://www.datapipe-js.com/docs/datapipe-js-array#transpose) (array) - Transpose rows to columns in an array 78 | - [**sort**](https://www.datapipe-js.com/docs/datapipe-js-array#sort) ([fieldName(s)]) - Sort array of elements according to a field and direction specified. e.g. sort(array, 'name ASC', 'age DESC') 79 | - [**flattenObject**](https://www.datapipe-js.com/docs/datapipe-js-array#flattenObject) (Object) - flattens complex nested object into simple object. e.g. flattenObject(obj) 80 | - [**unflattenObject**](https://www.datapipe-js.com/docs/datapipe-js-array#unflattenObject) (Object) - unflattens simple object into complex nested object. e.g. unflattenObject(obj) 81 | 82 | ### Joining data arrays 83 | 84 | - [**innerJoin**](https://www.datapipe-js.com/docs/datapipe-js-array#innerjoin) (leftArray, rightArray, leftKey, rightKey, resultSelector) - Joins two arrays together by selecting elements that have matching values in both arrays. The array elements that do not have matche in one array will not be shown! 85 | - [**leftJoin**](https://www.datapipe-js.com/docs/datapipe-js-array#leftjoin) (leftArray, rightArray, leftKey, rightKey, resultSelector) - Joins two arrays together by selrcting all elements from the left array (leftArray), and the matched elements from the right array (rightArray). The result is NULL from the right side, if there is no match. 86 | - [**fullJoin**](https://www.datapipe-js.com/docs/datapipe-js-array#fulljoin) (leftArray, rightArray, leftKey, rightKey, resultSelector) - Joins two arrays together by selrcting all elements from the left array (leftArray), and the matched elements from the right array (rightArray). The result is NULL from the right side, if there is no match. 87 | - [**merge**](https://www.datapipe-js.com/docs/datapipe-js-array#merge) (targetArray, sourceArray, targetKey, sourceKey) - merges elements from two arrays. It takes source elements and append or override elements in the target array.Merge or append is based on matching keys provided 88 | 89 | 90 | ### Aggregation and other numerical functions 91 | 92 | - [**avg**](https://www.datapipe-js.com/docs/datapipe-js-array#avg) ([propertySelector, predicate]) synonym **average** - returns an average value for a gived array. With `propertySelector` you can choose the property to calculate average on. And with `predicate` you can filter elements if needed. Both properties are optional. 93 | - [**max**](https://www.datapipe-js.com/docs/datapipe-js-array#max) ([propertySelector, predicate]) synonym **maximum** - returns a maximum value for a gived array. With `propertySelector` you can choose the property to calculate maximum on. And with `predicate` you can filter elements if needed. Both properties are optional. 94 | - [**min**](https://www.datapipe-js.com/docs/datapipe-js-array#min) ([propertySelector, predicate]) synonym **minimum** - returns a minimum value for a gived array. With `propertySelector` you can choose the property to calculate minimum on. And with `predicate` you can filter elements if needed. Both properties are optional. 95 | - [**count**](https://www.datapipe-js.com/docs/datapipe-js-array#count) ([predicate]) - returns the count for an elements in a pipe. With `predicate` function you can specify criteria 96 | - [**first**](https://www.datapipe-js.com/docs/datapipe-js-array#first) ([predicate]) - returns a first element in a pipe. If predicate function provided. Then it will return the first element in a pipe for a given criteria. 97 | - [**last**](https://www.datapipe-js.com/docs/datapipe-js-array#last) ([predicate]) - returns a first element in a pipe. If predicate function provided. Then it will return the first element in a pipe for a given criteria. 98 | - [**mean**](https://www.datapipe-js.com/docs/datapipe-js-array#mean) (array, [propertySelector]) - returns a mean in array. 99 | - [**quantile**](https://www.datapipe-js.com/docs/datapipe-js-array#quantile) array, [propertySelector]) - returns a quantile in array. 100 | - [**variance**](https://www.datapipe-js.com/docs/datapipe-js-array#variance) (array, [propertySelector]) - returns a sample variance of an array. 101 | - [**stdev**](https://www.datapipe-js.com/docs/datapipe-js-array#stdev) (array, [propertySelector]) - returns a standard deviation in array. 102 | - [**median**](https://www.datapipe-js.com/docs/datapipe-js-array#median) (array, [propertySelector]) - returns a median in array. 103 | 104 | ### Output your pipe data to 105 | 106 | - [**toArray**](https://www.datapipe-js.com/docs/datapipe-js-utils#toarray) - output your pipe result into JavaScript array. 107 | - [**toObject**](https://www.datapipe-js.com/docs/datapipe-js-utils#toobject) (nameSelector, valueSelector) - output your pipe result into JavaScript object, based of name and value selectors. 108 | - [**toSeries**](https://www.datapipe-js.com/docs/datapipe-js-utils#toseries) (propertyNames) - convert array into an object of series. 109 | - [**toCsv**](https://www.datapipe-js.com/docs/datapipe-js-utils#tocsv) ([delimiter]) - output pipe result into string formated as CSV 110 | when in browser. 111 | 112 | 113 | ### Other helpful utilities for working with data in JavaScript or JSPython 114 | - [**parseDatetimeOrNull**](https://www.datapipe-js.com/docs/datapipe-js-utils#parsedatetimeornull) (dateString[, formats]) - a bit wider date time parser than JS's `parseDate()`. Be aware. It gives UK time format (dd/MM/yyyy) a priority! e.g. '8/2/2019' will be parsed to 8th of February 2019 115 | - [**dateToString**](https://www.datapipe-js.com/docs/datapipe-js-utils#datetostring) (date, format) - converts date to string without applying time zone. It returns ISO formated date with time (if time present). Otherwise it will return just a date - yyyy-MM-dd 116 | - [**parseNumberOrNull**](https://www.datapipe-js.com/docs/datapipe-js-utils#parsenumberornull) (value: string | number): convert to number or returns null 117 | - [**parseBooleanOrNull**](https://www.datapipe-js.com/docs/datapipe-js-utils#parsebooleanornull) (val: boolean | string): convert to Boolean or returns null. It is treating `['1', 'yes', 'true', 'on']` as true and `['0', 'no', 'false', 'off']` as false 118 | - [**deepClone**](https://www.datapipe-js.com/docs/datapipe-js-utils#deepclone) returns a deep copy of your object or array. 119 | - [**getFieldsInfo**](https://www.datapipe-js.com/docs/datapipe-js-utils#getfieldsinfo) (items: Record[]): FieldDescription[] : Generates a field descriptions (first level only) from array of items. That eventually can be used for relational table definition. If any properties are Objects, it would use JSON.stringify to calculate maxSize field. 120 | - [**addDays**](https://www.datapipe-js.com/docs/datapipe-js-utils#adddays) (date: Date, daysOffset: number): Date: add days to the current date. `daysOffset` can be positive or negative number 121 | - [**addBusinessDays**](https://www.datapipe-js.com/docs/datapipe-js-utils#addbusinessdays) (date: Date, bDaysOffset: number): Date: Worksout a business date (excludes Saturdays and Sundays) based on bDaysOffset count. `bDaysOffset` can be positive or negative number. 122 | 123 | ### String Utils 124 | - [**replaceAll**](https://www.datapipe-js.com/docs/datapipe-js-string#replaceall) (text, searchValue, replaceValue) - Replace all string function 125 | - [**formatCamelStr**](https://www.datapipe-js.com/docs/datapipe-js-string#formatcamelstr) (text) - Formats string to the Camel Case 126 | - [**trimStart**](https://www.datapipe-js.com/docs/datapipe-js-string#trimstart) (text, charactersToTrim) - Trims characters from the start 127 | - [**trimEnd**](https://www.datapipe-js.com/docs/datapipe-js-string#trimend) (text, charactersToTrim) - Trims characters at the end 128 | - [**trim**](https://www.datapipe-js.com/docs/datapipe-js-string#trim) (text, charactersToTrim) - Trims characters in both sides 129 | - [**split**](https://www.datapipe-js.com/docs/datapipe-js-string#split) (text, separator, brackets): string - Splits text into tokens. Also, it supports multiple separators and will respect open/close brackets. e.g. `split('field1=func(a,b,c),field2=4', ',', ['()'])` will result into `["field1=func(a,b,c)", "field2=4"]` 130 | 131 | 132 | ## License 133 | A permissive [MIT](https://github.com/FalconSoft/dataPipe/blob/master/LICENSE) (c) - FalconSoft Ltd 134 | 135 | 136 | -------------------------------------------------------------------------------- /src/data-pipe.ts: -------------------------------------------------------------------------------- 1 | import { 2 | sum, 3 | avg, 4 | count, 5 | min, 6 | max, 7 | first, 8 | last, 9 | mean, 10 | quantile, 11 | variance, 12 | median, 13 | stdev 14 | } from './array/stats'; 15 | import { 16 | Selector, 17 | Predicate, 18 | ParsingOptions, 19 | FieldDescription, 20 | PrimitiveType, 21 | TableDto, 22 | DataTypeName, 23 | ScalarType 24 | } from './types'; 25 | import { parseCsv, fromTable, toTable, getFieldsInfo, toCsv, JSONParser } from './utils'; 26 | import { 27 | leftJoin, 28 | innerJoin, 29 | fullJoin, 30 | merge, 31 | groupBy, 32 | sort, 33 | pivot, 34 | transpose, 35 | toObject, 36 | toSeries, 37 | flatten, 38 | flattenObject, 39 | unflattenObject 40 | } from './array'; 41 | 42 | export class DataPipe { 43 | private data: T[]; 44 | 45 | constructor(data: T[] = []) { 46 | this.data = data || []; 47 | } 48 | 49 | // input methods 50 | fromCsv(content: string, options?: ParsingOptions): DataPipe { 51 | this.data = parseCsv(content, options) as unknown as T[]; 52 | return this; 53 | } 54 | 55 | parseCsv(content: string, options?: ParsingOptions): DataPipe { 56 | return this.fromCsv(content, options); 57 | } 58 | 59 | /** 60 | * Loads dataPipe with Table information 61 | * @param rowsOrTable a datarows with 2 dimentional arrays or entire table. If you provide rows, then you have to specify fieldNames 62 | * @param fieldNames fieldNames what correspond to the rows 63 | * @param fieldDataTypes fieldNames what correspond to the rows 64 | */ 65 | fromTable( 66 | rowsOrTable: PrimitiveType[][] | TableDto, 67 | fieldNames?: string[], 68 | fieldDataTypes?: DataTypeName[] 69 | ): DataPipe { 70 | this.data = fromTable(rowsOrTable, fieldNames, fieldDataTypes) as unknown as T[]; 71 | return this; 72 | } 73 | 74 | // Output methods 75 | /** 76 | * Get pipes currrent array data. 77 | */ 78 | toArray(): T[] { 79 | return this.data; 80 | } 81 | 82 | /** 83 | * Outputs Pipe value as CSV content 84 | * @param delimiter 85 | */ 86 | toCsv(delimiter = ','): string { 87 | return toCsv(this.data as unknown as Array>, delimiter); 88 | } 89 | 90 | /** 91 | * Outputs pipe value as JavaScript object. 92 | * @param keyField a key selector represented as a string (field name) or array of stringa (fieldNames) or custom selectors 93 | */ 94 | toObject(keyField: string | string[] | Selector): Record { 95 | return toObject(this.data, keyField); 96 | } 97 | 98 | /** 99 | * Convert array of items to into series array or series record. 100 | * @param propertyName optional parameter to define a property to be unpacked. 101 | * If it is string the array with values will be returned, otherwise an object with a list of series map 102 | */ 103 | toSeries(propertyName?: string | string[]): Record | ScalarType[] { 104 | return toSeries(this.data as unknown as Array>, propertyName); 105 | } 106 | 107 | /** 108 | * Gets table data from JSON type array. 109 | */ 110 | toTable(): TableDto { 111 | return toTable(this.data as unknown as Array>); 112 | } 113 | // end of output functions 114 | 115 | /** 116 | * This method allows you to examine a state of the data during pipe execution. 117 | * @param dataFunc 118 | */ 119 | tap(dataFunc: (d: T[]) => void): DataPipe { 120 | if (typeof dataFunc === 'function') { 121 | dataFunc(this.data); 122 | } 123 | return this; 124 | } 125 | 126 | // Aggregation Functions 127 | 128 | /** 129 | * Sum of items in array. 130 | * @param elementSelector Function invoked per iteration. 131 | * @example 132 | * dataPipe([1, 2, 5]).sum(); // 8 133 | * 134 | * dataPipe([{ val: 1 }, { val: 5 }]).sum(i => i.val); // 6 135 | */ 136 | sum(elementSelector?: Selector | string): number | null { 137 | return sum(this.data, elementSelector); 138 | } 139 | 140 | /** 141 | * Average of array items. 142 | * @param elementSelector Function invoked per iteration. 143 | * @example 144 | * dataPipe([1, 5, 3]).avg(); // 3 145 | */ 146 | avg(elementSelector?: Selector | string): number | null { 147 | return avg(this.data, elementSelector); 148 | } 149 | 150 | average = this.avg.bind(this); 151 | 152 | /** 153 | * Count of elements in array. 154 | * @param predicate Predicate function invoked per iteration. 155 | */ 156 | count(predicate?: Predicate): number | null { 157 | return count(this.data, predicate); 158 | } 159 | 160 | /** 161 | * Computes the minimum value of array. 162 | * @param elementSelector Function invoked per iteration. 163 | */ 164 | min(elementSelector?: Selector | string): number | Date | null { 165 | return min(this.data, elementSelector); 166 | } 167 | 168 | /** 169 | * Computes the maximum value of array. 170 | * @param elementSelector Function invoked per iteration. 171 | */ 172 | max(elementSelector?: Selector | string): number | Date | null { 173 | return max(this.data, elementSelector); 174 | } 175 | 176 | /** 177 | * Gets first item in array satisfies predicate. 178 | * @param predicate Predicate function invoked per iteration. 179 | */ 180 | first(predicate?: Predicate): T | null | undefined { 181 | return first(this.data, predicate); 182 | } 183 | 184 | /** 185 | * Gets last item in array satisfies predicate. 186 | * @param array The array to process. 187 | * @param predicate Predicate function invoked per iteration. 188 | */ 189 | last(predicate?: Predicate): T | null | undefined { 190 | return last(this.data, predicate); 191 | } 192 | 193 | /** 194 | * Get mean. 195 | * @param field Property name or Selector function invoked per iteration. 196 | */ 197 | mean(field?: Selector | string): number | null { 198 | return mean(this.data, field); 199 | } 200 | 201 | /** 202 | * Get quantile of a sorted array. 203 | * @param field Property name or Selector function invoked per iteration. 204 | * @param p quantile. 205 | */ 206 | quantile(p: number, field?: Selector | string): number | null { 207 | return quantile(this.data, p, field); 208 | } 209 | 210 | /** 211 | * Get variance. 212 | * @param field Property name or Selector function invoked per iteration. 213 | */ 214 | variance(field?: Selector | string): number | null { 215 | return variance(this.data, field); 216 | } 217 | 218 | /** 219 | * Get the standard deviation. 220 | * @param field Property name or Selector function invoked per iteration. 221 | */ 222 | stdev(field?: Selector | string): number | null { 223 | return stdev(this.data, field); 224 | } 225 | 226 | /** 227 | * Get median. 228 | * @param field Property name or Selector function invoked per iteration. 229 | */ 230 | median(field?: Selector | string): number | null { 231 | return median(this.data, field); 232 | } 233 | 234 | // Data Transformation functions 235 | /** 236 | * Groups array items based on elementSelector function 237 | * @param elementSelector Function invoked per iteration. 238 | */ 239 | groupBy(elementSelector: string | string[] | Selector): DataPipe { 240 | this.data = groupBy(this.data, elementSelector as string | string[] | Selector) as unknown as T[]; 241 | return this as unknown as DataPipe; 242 | } 243 | 244 | /** 245 | * Joins two arrays together by selecting elements that have matching values in both arrays 246 | * @param rightArray array of elements to join 247 | * @param leftKey left Key 248 | * @param rightKey 249 | * @param resultSelector 250 | */ 251 | innerJoin( 252 | rightArray: TRight[], 253 | leftKey: string | string[] | Selector, 254 | rightKey: string | string[] | Selector, 255 | resultSelector: (leftItem: T | null, rightItem: TRight | null) => TResult 256 | ): DataPipe { 257 | this.data = innerJoin(this.data, rightArray, leftKey, rightKey, resultSelector) as unknown as T[]; 258 | return this as unknown as DataPipe; 259 | } 260 | 261 | leftJoin( 262 | rightArray: TRight[], 263 | leftKey: string | string[] | Selector, 264 | rightKey: string | string[] | Selector, 265 | resultSelector: (leftItem: T | null, rightItem: TRight | null) => TResult 266 | ): DataPipe { 267 | this.data = leftJoin(this.data, rightArray, leftKey, rightKey, resultSelector) as unknown as T[]; 268 | return this as unknown as DataPipe; 269 | } 270 | 271 | fullJoin( 272 | rightArray: TRight[], 273 | leftKey: string | string[] | Selector, 274 | rightKey: string | string[] | Selector, 275 | resultSelector: (leftItem: T | null, rightItem: TRight | null) => TResult 276 | ): DataPipe { 277 | this.data = fullJoin(this.data, rightArray, leftKey, rightKey, resultSelector) as unknown as T[]; 278 | return this as unknown as DataPipe; 279 | } 280 | 281 | /** 282 | * Select data from array item. 283 | * @param elementSelector Function invoked per iteration. 284 | * @example 285 | * 286 | * dataPipe([{ val: 1 }, { val: 5 }]).select(i => i.val).toArray(); // [1, 5] 287 | */ 288 | select(elementSelector: Selector): DataPipe { 289 | this.data = this.data.map(elementSelector) as unknown as T[]; 290 | return this as unknown as DataPipe; 291 | } 292 | 293 | map = this.select.bind(this); 294 | 295 | merge( 296 | sourceArray: TSource[], 297 | targetKey: string | string[] | Selector, 298 | sourceKey: string | string[] | Selector 299 | ): DataPipe { 300 | this.data = merge(this.data, sourceArray, targetKey, sourceKey); 301 | return this; 302 | } 303 | 304 | flattenObject(): unknown { 305 | return flattenObject(this.data); 306 | } 307 | 308 | unflattenObject(): unknown { 309 | return unflattenObject(this.data); 310 | } 311 | 312 | flatten(): unknown { 313 | return flatten(this.data); 314 | } 315 | 316 | pivot( 317 | rowFields: string | string[], 318 | columnField: string, 319 | dataField: string, 320 | aggFunction?: (array: unknown[]) => unknown | null, 321 | columnValues?: string[] 322 | ): DataPipe> { 323 | this.data = pivot(this.data as Array>, rowFields, columnField, dataField, aggFunction, columnValues) as unknown as T[]; 324 | return this as unknown as DataPipe>; 325 | } 326 | 327 | transpose(): DataPipe> { 328 | this.data = transpose(this.data as Array>) as unknown as T[] || [] as unknown as T[]; 329 | return this as unknown as DataPipe>; 330 | } 331 | 332 | /** 333 | * Filters array of items. 334 | * @param predicate Predicate function invoked per iteration. 335 | */ 336 | where(predicate: Predicate): DataPipe { 337 | this.data = this.data.filter(predicate); 338 | return this; 339 | } 340 | 341 | filter = this.where.bind(this); 342 | 343 | /** 344 | * Get unique values 345 | * @param elementSelector Function invoked per iteration. 346 | */ 347 | unique(elementSelector?: Selector): DataPipe { 348 | if (elementSelector) { 349 | this.select(elementSelector); 350 | } 351 | this.data = Array.from(new Set(this.data)); 352 | return this as unknown as DataPipe; 353 | } 354 | 355 | distinct = this.unique.bind(this); 356 | 357 | /** 358 | * Sort array. 359 | * @param fields sorts order. 360 | * @example 361 | * dp.sort('name ASC', 'age DESC'); 362 | */ 363 | sort(...fields: string[]): DataPipe { 364 | sort(this.data, ...fields); 365 | return this; 366 | } 367 | 368 | // end of transformation functions 369 | 370 | /** 371 | * generates a field descriptions (first level only) that can be used for relational table definition. 372 | * if any properties are Objects, it would use JSON.stringify to calculate maxSize field. 373 | */ 374 | getFieldsInfo(): FieldDescription[] { 375 | return getFieldsInfo(this.data as unknown as Array>); 376 | } 377 | 378 | JSONParser(): JSONParser { 379 | return new JSONParser(); 380 | } 381 | } 382 | -------------------------------------------------------------------------------- /src/tests/dsv-parser.spec.ts: -------------------------------------------------------------------------------- 1 | import { parseCsv, toCsv, parseCsvToTable, dateToString } from '../utils'; 2 | import { ParsingOptions, DataTypeName } from '../types'; 3 | 4 | describe('Dsv Parser specification', () => { 5 | it('simple numbers', () => { 6 | const csv = ['F1,F2', '1,2'].join('\n'); 7 | const result = parseCsv(csv); 8 | expect(result.length).toBe(1); 9 | expect(result[0].F1).toBe(1); 10 | expect(result[0].F2).toBe(2); 11 | }); 12 | 13 | it('simple numbers (double)', () => { 14 | const csv = ['F1,F2', '1,2.5'].join('\n'); 15 | const result = parseCsv(csv); 16 | expect(result.length).toBe(1); 17 | expect(result[0].F1).toBe(1); 18 | expect(result[0].F2).toBe(2.5); 19 | }); 20 | 21 | it('simple numbers (double) with thousand', () => { 22 | const csv = ['F1,F2', `1,"2,000.5"`].join('\n'); 23 | const result = parseCsv(csv); 24 | expect(result.length).toBe(1); 25 | expect(result[0].F1).toBe(1); 26 | expect(result[0].F2).toBe(2000.5); 27 | }); 28 | 29 | it('simple numbers (double) negative', () => { 30 | const csv = ['F1,F2', `1,-2000.5`].join('\n'); 31 | const result = parseCsv(csv); 32 | expect(result.length).toBe(1); 33 | expect(result[0].F1).toBe(1); 34 | expect(result[0].F2).toBe(-2000.5); 35 | }); 36 | 37 | it('simple numbers (double) with thousand', () => { 38 | const csv = ['F1,F2', `1,"-2,000.5"`].join('\n'); 39 | const result = parseCsv(csv); 40 | expect(result.length).toBe(1); 41 | expect(result[0].F1).toBe(1); 42 | expect(result[0].F2).toBe(-2000.5); 43 | }); 44 | 45 | it('simple numders and strings', () => { 46 | const csv = ['F1,F2,F3', `1,2,"Test, comma"`].join('\n'); 47 | const result = parseCsv(csv); 48 | expect(result.length).toBe(1); 49 | expect(result[0].F3).toBe('Test, comma'); 50 | }); 51 | 52 | it('String with quotes 1', () => { 53 | const csv = ['F1,F2', `1,"T ""k"" c"`].join('\n'); 54 | const result = parseCsv(csv); 55 | expect(result.length).toBe(1); 56 | expect(result[0].F2).toBe('T "k" c'); 57 | }); 58 | 59 | it('String with quotes 2', () => { 60 | const csv = ['F1,F2,F3', `1,"T ""k"" c",77`].join('\n'); 61 | const result = parseCsv(csv); 62 | expect(result.length).toBe(1); 63 | expect(result[0].F2).toBe('T "k" c'); 64 | expect(result[0].F3).toBe(77); 65 | }); 66 | 67 | it('String with quotes 3', () => { 68 | const csv = ['F1,F2,F3', `1,"T """,77`].join('\n'); 69 | const result = parseCsv(csv); 70 | expect(result.length).toBe(1); 71 | expect(result[0].F2).toBe('T "'); 72 | expect(result[0].F3).toBe(77); 73 | }); 74 | 75 | it('String with quotes 4', () => { 76 | const csv = ['F1,F2', `1,"T """`].join('\n'); 77 | const result = parseCsv(csv); 78 | expect(result.length).toBe(1); 79 | expect(result[0].F2).toBe('T "'); 80 | }); 81 | 82 | it('String with quotes 5 empty " " ', () => { 83 | const csv = ['F1,F2', `1," "`].join('\n'); 84 | const result = parseCsv(csv); 85 | expect(result.length).toBe(1); 86 | expect(result[0].F2).toBe(' '); 87 | }); 88 | 89 | it('String with quotes 6 empty " " ', () => { 90 | const csv = ['F1,F2', `1," "`].join('\n'); 91 | const result = parseCsv(csv); 92 | expect(result.length).toBe(1); 93 | expect(result[0].F2).toBe(' '); 94 | }); 95 | 96 | it('String with quotes 7 empty "" ', () => { 97 | const csv = ['F1,F2', `1,"dd"`, `1,""`].join('\n'); 98 | const result = parseCsv(csv); 99 | expect(result[1].F2).toBe(''); 100 | }); 101 | 102 | it('simple numders and strings with spaces', () => { 103 | const csv = ['F1,F2 ,F3', `1,2,"Test, comma"`].join('\n'); 104 | const result = parseCsv(csv); 105 | expect(result.length).toBe(1); 106 | expect(result[0].F2).toBe(2); 107 | }); 108 | 109 | it('simple numders and zeros', () => { 110 | const csv = ['F1,F2,F3', `0,2,0`].join('\n'); 111 | const result = parseCsv(csv); 112 | expect(result.length).toBe(1); 113 | expect(result[0].F1).toBe(0); 114 | expect(result[0].F2).toBe(2); 115 | expect(result[0].F3).toBe(0); 116 | }); 117 | 118 | it('Empty should be null', () => { 119 | const csv = ['F1,F2,F3', `1,,"Test, comma"`].join('\n'); 120 | const result = parseCsv(csv); 121 | expect(result.length).toBe(1); 122 | expect(result[0].F2).toBe(null); 123 | }); 124 | 125 | it('Multiline string', () => { 126 | const multiLineString = `this is , 5 - , 127 | multi-line 128 | string`; 129 | const csv = ['F1,F2,F3', `1,"${multiLineString}","Test, comma"`].join('\n'); 130 | const result = parseCsv(csv); 131 | expect(result.length).toBe(1); 132 | expect(result[0].F2).toBe(multiLineString); 133 | }); 134 | 135 | it('Multiline string DSV', () => { 136 | const multiLineString = `this is , 5 - , 137 | multi-line 138 | string`; 139 | const csv = ['F1\tF2\tF3', `1\t"${multiLineString}"\t"Test, comma"`].join('\n'); 140 | const result = parseCsv(csv, { delimiter: '\t' } as ParsingOptions); 141 | expect(result.length).toBe(1); 142 | expect(result[0].F2).toBe(multiLineString); 143 | }); 144 | 145 | it('DSV with comma numbers', () => { 146 | const csv = ['F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`].join('\n'); 147 | const result = parseCsv(csv, { delimiter: '\t' } as ParsingOptions); 148 | expect(result.length).toBe(1); 149 | expect(result[0].F2).toBe(1000.32); 150 | }); 151 | 152 | it('skip rows', () => { 153 | const csv = ['', '', 'F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`].join('\n'); 154 | const result = parseCsv(csv, { delimiter: '\t', skipRows: 2 } as ParsingOptions); 155 | expect(result.length).toBe(1); 156 | expect(result[0].F2).toBe(1000.32); 157 | }); 158 | 159 | it('skip rows not empty rows', () => { 160 | const csv = ['', ' * not Empty *', 'F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`].join('\n'); 161 | const result = parseCsv(csv, { delimiter: '\t', skipRows: 2 } as ParsingOptions); 162 | expect(result.length).toBe(1); 163 | expect(result[0].F2).toBe(1000.32); 164 | }); 165 | 166 | it('skipUntil', () => { 167 | const csv = ['', ' * not Empty *', 'F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`].join('\n'); 168 | const options = new ParsingOptions(); 169 | options.delimiter = '\t'; 170 | options.skipUntil = (t: string[]): boolean => t && t.length > 1; 171 | 172 | const result = parseCsv(csv, options); 173 | expect(result.length).toBe(1); 174 | expect(result[0].F2).toBe(1000.32); 175 | }); 176 | 177 | it('empty values', () => { 178 | const csv = ['', '', '\t\t\t', 'F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`, '\t\t'].join('\n'); 179 | const options = new ParsingOptions(); 180 | options.delimiter = '\t'; 181 | 182 | const result = parseCsv(csv, options); 183 | expect(result.length).toBe(1); 184 | expect(result[0].F2).toBe(1000.32); 185 | }); 186 | 187 | it('Date Fields', () => { 188 | const csv = ['F1\tF2\tF3', `2020-02-11\t1,000.32\t"Test, comma"`].join('\n'); 189 | const options = new ParsingOptions(); 190 | options.delimiter = '\t'; 191 | options.dateFields = ['F1']; 192 | 193 | const result = parseCsv(csv, options); 194 | expect(result.length).toBe(1); 195 | expect(result[0].F1 instanceof Date).toBe(true); 196 | }); 197 | 198 | it('Still string. Because second row is not a number or date', () => { 199 | const csv = ['F1\tF2\tF3', `2020-02-11\t1,000.32\t"Test, comma"`, 'nn\tnn\tnn'].join('\n'); 200 | const options = new ParsingOptions(); 201 | options.delimiter = '\t'; 202 | 203 | const result = parseCsv(csv, options); 204 | expect(result.length).toBe(2); 205 | expect(typeof result[0].F1 === 'string').toBe(true); 206 | expect(typeof result[0].F2 === 'string').toBe(true); 207 | expect(result[0].F2).toBe('1,000.32'); 208 | }); 209 | 210 | it('ToCsv', () => { 211 | const csv = ['F1,F2,F3', `2020-02-11,1,tt`].join('\n'); 212 | const result = toCsv(parseCsv(csv)); 213 | expect(result).toBe(csv); 214 | }); 215 | }); 216 | 217 | describe('Parse Csv To Table', () => { 218 | it('simple numbers', () => { 219 | const csv = ['F1,F2', '1,2'].join('\n'); 220 | const result = parseCsvToTable(csv); 221 | expect(result.rows.length).toBe(1); 222 | expect(result.fieldDescriptions.length).toBe(2); 223 | expect(result.fieldDescriptions[0].fieldName).toBe('F1'); 224 | expect(result.fieldDescriptions[0].dataTypeName).toBe(DataTypeName.WholeNumber); 225 | expect(result.fieldDescriptions[0].isUnique).toBe(true); 226 | expect(result.fieldDescriptions[0].isNullable).toBe(false); 227 | // everything returns as a string 228 | expect(result.rows[0][result.fieldDescriptions[0].index]).toBe('1'); 229 | }); 230 | 231 | it('double and non unique', () => { 232 | const csv = ['F1,F2', '1,2', '1.3,2'].join('\n'); 233 | const result = parseCsvToTable(csv); 234 | expect(result.rows.length).toBe(2); 235 | expect(result.fieldDescriptions.length).toBe(2); 236 | expect(result.fieldDescriptions[0].fieldName).toBe('F1'); 237 | expect(result.fieldDescriptions[0].dataTypeName).toBe(DataTypeName.FloatNumber); 238 | expect(result.fieldDescriptions[0].isUnique).toBe(true); 239 | expect(result.fieldDescriptions[1].isUnique).toBe(false); 240 | expect(result.fieldDescriptions[0].isNullable).toBe(false); 241 | // everything returns as a string 242 | expect(result.rows[0][result.fieldDescriptions[0].index]).toBe('1'); 243 | }); 244 | 245 | it('Date parse', () => { 246 | const csv = ['F1,F2', '1,06/02/2020', '1.3,06/07/2020'].join('\n'); 247 | const result = parseCsv(csv, { dateFields: [['F2', 'MM/dd/yyyy']] } as ParsingOptions); 248 | expect(result.length).toBe(2); 249 | expect(dateToString(result[0].F2 as Date)).toBe('2020-06-02'); 250 | expect(dateToString(result[1].F2 as Date)).toBe('2020-06-07'); 251 | }); 252 | 253 | it('Date parse 2', () => { 254 | const csv = ['F1,F2', '1,20200602', '1.3,20200607'].join('\n'); 255 | const result = parseCsv(csv, { dateFields: [['F2', 'yyyyMMdd']] } as ParsingOptions); 256 | expect(result.length).toBe(2); 257 | expect(dateToString(result[0].F2 as Date)).toBe('2020-06-02'); 258 | expect(dateToString(result[1].F2 as Date)).toBe('2020-06-07'); 259 | }); 260 | 261 | it('Date parse 3', () => { 262 | const csv = ['F1,F2', '1,202006', '1.3,202005'].join('\n'); 263 | const result = parseCsv(csv, { dateFields: [['F2', 'yyyyMM']] } as ParsingOptions); 264 | expect(result.length).toBe(2); 265 | expect(dateToString(result[0].F2 as Date)).toBe('2020-06-01'); 266 | expect(dateToString(result[1].F2 as Date)).toBe('2020-05-01'); 267 | }); 268 | 269 | it('Date parse 4', () => { 270 | const csv = ['F1,F2', '1,06/02/2020', '1.3,06/07/2020'].join('\n'); 271 | const result = parseCsv(csv, { dateFields: [['F2', 'dd/MM/yyyy']] } as ParsingOptions); 272 | expect(result.length).toBe(2); 273 | expect(dateToString(result[0].F2 as Date)).toBe('2020-02-06'); 274 | expect(dateToString(result[1].F2 as Date)).toBe('2020-07-06'); 275 | }); 276 | 277 | it('boolean test', () => { 278 | const csv = ['F1,F2', 'true,2', 'TRUE,2'].join('\n'); 279 | const result = parseCsvToTable(csv); 280 | expect(result.rows.length).toBe(2); 281 | expect(result.fieldDescriptions.length).toBe(2); 282 | expect(result.fieldDescriptions[0].fieldName).toBe('F1'); 283 | expect(result.fieldDescriptions[0].dataTypeName).toBe(DataTypeName.Boolean); 284 | expect(result.fieldDescriptions[0].isUnique).toBe(true); 285 | expect(result.fieldDescriptions[1].isUnique).toBe(false); 286 | expect(result.fieldDescriptions[0].isNullable).toBe(false); 287 | // everything returns as a string 288 | expect(result.rows[0][result.fieldDescriptions[0].index]).toBe('true'); 289 | expect(result.rows[1][result.fieldDescriptions[0].index]).toBe('TRUE'); 290 | }); 291 | 292 | it('nullable boolean test', () => { 293 | const csv = ['F1,F2', 'true,2', ',2'].join('\n'); 294 | const result = parseCsvToTable(csv); 295 | expect(result.rows.length).toBe(2); 296 | expect(result.fieldDescriptions[0].isNullable).toBe(true); 297 | expect(result.rows[0][result.fieldDescriptions[0].index]).toBe('true'); 298 | expect(result.rows[1][result.fieldDescriptions[0].index]).toBe(null); 299 | }); 300 | 301 | it('nullable date test', () => { 302 | const csv = ['F1,F2', '2020-02-02,2', ',2'].join('\n'); 303 | const result = parseCsvToTable(csv); 304 | expect(result.rows.length).toBe(2); 305 | expect(result.fieldDescriptions[0].isNullable).toBe(true); 306 | expect(result.fieldDescriptions[0].dataTypeName).toBe(DataTypeName.Date); 307 | 308 | expect(result.rows[0][result.fieldDescriptions[0].index]).toBe('2020-02-02'); 309 | expect(result.rows[1][result.fieldDescriptions[0].index]).toBe(null); 310 | }); 311 | 312 | it('nullable date test 2', () => { 313 | const csv = ['F1,F2', ',2', '2020-02-02,2'].join('\n'); 314 | const result = parseCsvToTable(csv); 315 | expect(result.rows.length).toBe(2); 316 | expect(result.fieldDescriptions[0].isNullable).toBe(true); 317 | expect(result.fieldDescriptions[0].dataTypeName).toBe(DataTypeName.Date); 318 | 319 | expect(result.rows[1][result.fieldDescriptions[0].index]).toBe('2020-02-02'); 320 | expect(result.rows[0][result.fieldDescriptions[0].index]).toBe(null); 321 | }); 322 | 323 | it('nullable float test', () => { 324 | const csv = ['F1,F2', ',2', ',-2020.98'].join('\n'); 325 | const result = parseCsvToTable(csv); 326 | expect(result.rows.length).toBe(2); 327 | expect(result.fieldDescriptions[0].isNullable).toBe(true); 328 | expect(result.fieldDescriptions[0].dataTypeName || null).toBe(null); 329 | expect(result.fieldDescriptions[1].dataTypeName).toBe(DataTypeName.FloatNumber); 330 | 331 | expect(result.rows[1][result.fieldDescriptions[1].index]).toBe('-2020.98'); 332 | expect(result.rows[0][result.fieldDescriptions[0].index]).toBe(null); 333 | }); 334 | 335 | it('header smoothing', () => { 336 | const csv = ['F 1,F 2', '11,12', '21,22'].join('\n'); 337 | const result = parseCsvToTable(csv); 338 | expect(result.fieldNames.length).toBe(2); 339 | expect(result.fieldNames[0]).toBe('F_1'); 340 | expect(result.fieldNames[1]).toBe('F_2'); 341 | }); 342 | 343 | it('toCsv/parse', () => { 344 | expect(toCsv(parseCsv('F 1,F 2\n11,12\n21,22'))).toBe('F_1,F_2\n11,12\n21,22'); 345 | expect( 346 | toCsv(parseCsv('F 1,F 2\n11,12\n21,22', { keepOriginalHeaders: true } as ParsingOptions)) 347 | ).toBe('F 1,F 2\n11,12\n21,22'); 348 | }); 349 | 350 | it('toCsv', () => { 351 | const obj = [ 352 | { f1: 1, f2: 'test' }, 353 | { f1: 2, f2: 'test"' }, 354 | { f1: 3, f2: 'te,st' } 355 | ]; 356 | expect(toCsv(obj)).toBe(`f1,f2\n1,test\n2,"test"""\n3,"te,st"`); 357 | }); 358 | }); 359 | -------------------------------------------------------------------------------- /src/tests/array.spec.ts: -------------------------------------------------------------------------------- 1 | import * as pipeFuncs from '../array'; 2 | import { 3 | leftJoin, 4 | pivot, 5 | avg, 6 | sum, 7 | quantile, 8 | mean, 9 | variance, 10 | stdev, 11 | median, 12 | first, 13 | fullJoin, 14 | innerJoin, 15 | toObject, 16 | toSeries 17 | } from '../array'; 18 | 19 | export const data: Array<{ name: string; country: string; age?: number | null }> = [ 20 | { name: 'John', country: 'US', age: 32 }, 21 | { name: 'Joe', country: 'US', age: 24 }, 22 | { name: 'Bill', country: 'US', age: 27 }, 23 | { name: 'Adam', country: 'UK', age: 18 }, 24 | { name: 'Scott', country: 'UK', age: 45 }, 25 | { name: 'Diana', country: 'UK' }, 26 | { name: 'Marry', country: 'FR', age: 18 }, 27 | { name: 'Luc', country: 'FR', age: null } 28 | ]; 29 | 30 | describe('Test array methods', () => { 31 | const testNumberArray = [2, 6, 3, 7, 11, 7, -1]; 32 | const testNumberArraySum = 35; 33 | const testNumberArrayAvg = 5; 34 | 35 | const testAnyPrimitiveArray = ['5', 2, '33', false, true, true, true]; 36 | const testAnyPrimitiveArraySum = 43; 37 | const testAnyPrimitiveArrayAvg = 6.14; 38 | const testAnyPrimitiveArrayMin = 0; 39 | const testAnyPrimitiveArrayMax = 33; 40 | 41 | const testObjArray: Array<{ value: number }> = testNumberArray.map(value => ({ value })); 42 | const dates = [ 43 | new Date('10/01/12'), 44 | new Date('10/01/10'), 45 | new Date('10/01/09'), 46 | new Date('10/01/11') 47 | ]; 48 | 49 | it('count', () => { 50 | expect(pipeFuncs.count(testNumberArray)).toBe(testNumberArray.length); 51 | expect(pipeFuncs.count(testAnyPrimitiveArray)).toBe(testAnyPrimitiveArray.length); 52 | expect(pipeFuncs.count(testObjArray)).toBe(testObjArray.length); 53 | expect(pipeFuncs.count(data, (i: unknown) => (i as (typeof data)[0]).country === 'US')).toBe(3); 54 | }); 55 | 56 | it('sum', () => { 57 | expect(pipeFuncs.sum(testNumberArray)).toBe(testNumberArraySum); 58 | expect(pipeFuncs.sum(testAnyPrimitiveArray)).toBe(testAnyPrimitiveArraySum); 59 | expect( 60 | pipeFuncs.sum( 61 | testObjArray, 62 | obj => (obj as (typeof testObjArray)[0]).value as unknown as string 63 | ) 64 | ).toBe(testNumberArraySum); 65 | }); 66 | 67 | it('avg', () => { 68 | expect(pipeFuncs.avg(testNumberArray)).toBe(testNumberArrayAvg); 69 | expect(pipeFuncs.avg(testObjArray, obj => obj.value as unknown as string)).toBe( 70 | testNumberArrayAvg 71 | ); 72 | 73 | const avg = pipeFuncs.avg(testAnyPrimitiveArray); 74 | if (avg) { 75 | expect(Math.round(avg * 100) / 100).toBe(testAnyPrimitiveArrayAvg); 76 | } else { 77 | throw Error('testAnyPrimitiveArray failed'); 78 | } 79 | }); 80 | 81 | it('min', () => { 82 | expect(pipeFuncs.min(testNumberArray)).toBe(Math.min(...testNumberArray)); 83 | expect(pipeFuncs.min(testAnyPrimitiveArray)).toBe(testAnyPrimitiveArrayMin); 84 | expect( 85 | pipeFuncs.min( 86 | testObjArray, 87 | (obj: unknown) => (obj as (typeof testObjArray)[0]).value as unknown as string 88 | ) 89 | ).toBe(Math.min(...testNumberArray)); 90 | 91 | const mindate = pipeFuncs.min(dates); 92 | expect(mindate).toBeInstanceOf(Date); 93 | if (mindate instanceof Date) { 94 | expect(mindate.getFullYear()).toBe(2009); 95 | } 96 | }); 97 | 98 | it('max', () => { 99 | expect(pipeFuncs.max(testNumberArray)).toBe(Math.max(...testNumberArray)); 100 | expect(pipeFuncs.max(testAnyPrimitiveArray)).toBe(testAnyPrimitiveArrayMax); 101 | expect( 102 | pipeFuncs.max( 103 | testObjArray, 104 | (obj: unknown) => (obj as (typeof testObjArray)[0]).value as unknown as string 105 | ) 106 | ).toBe(Math.max(...testNumberArray)); 107 | expect(pipeFuncs.max([])).toBe(null); 108 | const maxdate = pipeFuncs.max(dates); 109 | expect(maxdate).toBeInstanceOf(Date); 110 | if (maxdate instanceof Date) { 111 | expect(maxdate.getFullYear()).toBe(2012); 112 | } 113 | }); 114 | 115 | it('first', () => { 116 | expect(pipeFuncs.first(testNumberArray)).toBe(testNumberArray[0]); 117 | expect(pipeFuncs.first(testNumberArray, (v: unknown) => (v as number) > 6)).toBe(7); 118 | }); 119 | 120 | it('last', () => { 121 | const last = pipeFuncs.last( 122 | data, 123 | (item: unknown) => (item as (typeof data)[0]).country === 'UK' 124 | ); 125 | if (last) { 126 | expect(last.name).toBe('Diana'); 127 | } else { 128 | throw Error('Not found'); 129 | } 130 | }); 131 | 132 | it('groupBy', () => { 133 | const groups = pipeFuncs.groupBy(data, (item: unknown) => (item as (typeof data)[0]).country); 134 | expect(groups.length).toBe(3); 135 | }); 136 | 137 | it('flatten', () => { 138 | const testArray = [1, 4, [2, [5, 5, [9, 7]], 11], 0, [], []]; 139 | const flatten = pipeFuncs.flatten(testArray); 140 | expect(flatten.length).toBe(9); 141 | 142 | const testArray2 = [testArray, [data], [], [testAnyPrimitiveArray]]; 143 | const flatten2 = pipeFuncs.flatten(testArray2); 144 | expect(flatten2.length).toBe(9 + data.length + testAnyPrimitiveArray.length); 145 | }); 146 | 147 | it('flattenObject', () => { 148 | const testArray = [ 149 | { a: 1, d: { d1: 22, d2: 33 } }, 150 | { b: 2, d: { d1: 221, d2: 331 } } 151 | ]; 152 | const flatten = pipeFuncs.flattenObject(testArray); 153 | expect(flatten.length).toBe(2); 154 | expect(Object.keys(flatten[0]).join(',')).toBe('a,d.d1,d.d2'); 155 | }); 156 | 157 | it('unflattenObject', () => { 158 | const testArray = [ 159 | { a: 1, 'b.e': 2, 'b.c.d': 2, 'b.c.f': 3, 'b.f': 5 }, 160 | { a: -1, 'b.e': -2, 'b.c.d': -2, 'b.c.f': -3, 'b.f': -5 } 161 | ]; 162 | const unflatten = pipeFuncs.unflattenObject(testArray) as Record[]; 163 | expect(unflatten.length).toBe(2); 164 | expect(Object.keys(unflatten[0]).join(',')).toBe('a,b'); 165 | // expect(unflatten[0].b.c['d']).toBe(2); 166 | // expect(unflatten[1].b.c['d']).toBe(-2); 167 | }); 168 | 169 | it('countBy', () => { 170 | const countriesCount = pipeFuncs.countBy(data, (i: unknown) => (i as (typeof data)[0]).country); 171 | expect(countriesCount['US']).toBe(3); 172 | expect(countriesCount['UK']).toBe(3); 173 | expect(countriesCount['FR']).toBe(2); 174 | }); 175 | 176 | it('handle empty arrays', () => { 177 | expect(pipeFuncs.max([])).toBe(null); 178 | expect(pipeFuncs.max([null])).toBe(null); 179 | expect(pipeFuncs.max([undefined])).toBe(null); 180 | expect(pipeFuncs.min([])).toBe(null); 181 | expect(pipeFuncs.min([null])).toBe(null); 182 | expect(pipeFuncs.min([undefined])).toBe(null); 183 | expect(pipeFuncs.avg([])).toBe(null); 184 | expect(pipeFuncs.avg([null])).toBe(null); 185 | expect(pipeFuncs.avg([undefined])).toBe(null); 186 | expect(pipeFuncs.stdev([])).toBe(null); 187 | expect(pipeFuncs.first([])).toBe(null); 188 | expect(pipeFuncs.last([])).toBe(null); 189 | expect(pipeFuncs.mean([])).toBe(null); 190 | expect(pipeFuncs.median([])).toBe(null); 191 | expect(pipeFuncs.sum([])).toBe(null); 192 | expect(pipeFuncs.variance([])).toBe(null); 193 | expect(pipeFuncs.quantile([], 0)).toBe(null); 194 | }); 195 | 196 | it('leftJoin', () => { 197 | const countries = [ 198 | { code: 'US', capital: 'Washington' }, 199 | { code: 'UK', capital: 'London' } 200 | ]; 201 | const joinedArray = leftJoin( 202 | data, 203 | countries, 204 | i => i.country, 205 | i2 => i2.code, 206 | (l, r) => ({ ...r, ...l }) 207 | ); 208 | expect(joinedArray.length).toBe(8); 209 | const item = pipeFuncs.first( 210 | joinedArray, 211 | (i: unknown) => (i as { country: string }).country === 'US' 212 | ); 213 | if (item) { 214 | expect((item as { country: string }).country).toBe('US'); 215 | expect((item as { code: string }).code).toBe('US'); 216 | expect((item as { capital: string }).capital).toBe('Washington'); 217 | expect((item as { name: string }).name).toBeTruthy(); 218 | } 219 | }); 220 | 221 | it('leftJoin 2', () => { 222 | const arr1: { keyField: string; value1: number }[] = [ 223 | { keyField: 'k1', value1: 21 }, 224 | { keyField: 'k2', value1: 22 }, 225 | { keyField: 'k3', value1: 23 }, 226 | { keyField: 'k4', value1: 24 } 227 | ]; 228 | 229 | const arr2: { keyField: string; value2: number }[] = [ 230 | { keyField: 'k1', value2: 31 }, 231 | { keyField: 'k2', value2: 32 }, 232 | { keyField: 'k5', value2: 35 }, 233 | { keyField: 'k8', value2: 38 } 234 | ]; 235 | 236 | const joinedArray = leftJoin( 237 | arr1, 238 | arr2, 239 | l => l.keyField, 240 | r => r.keyField, 241 | (l, r) => ({ ...r, ...l } as { keyField: string; value1: number; value2: number }) 242 | ); 243 | expect(joinedArray.length).toBe(4); 244 | expect(joinedArray[1].keyField).toBe('k2'); 245 | expect(joinedArray[1].value1).toBe(22); 246 | expect(joinedArray[1].value2).toBe(32); 247 | }); 248 | 249 | it('innerJoin', () => { 250 | const arr1: { keyField: string; value1: number }[] = [ 251 | { keyField: 'k1', value1: 21 }, 252 | { keyField: 'k2', value1: 22 }, 253 | { keyField: 'k3', value1: 23 }, 254 | { keyField: 'k4', value1: 24 } 255 | ]; 256 | 257 | const arr2: { keyField: string; value2: number }[] = [ 258 | { keyField: 'k1', value2: 31 }, 259 | { keyField: 'k2', value2: 32 }, 260 | { keyField: 'k5', value2: 35 }, 261 | { keyField: 'k8', value2: 38 } 262 | ]; 263 | 264 | const joinedArray = innerJoin( 265 | arr1, 266 | arr2, 267 | l => l.keyField, 268 | r => r.keyField, 269 | (l, r) => ({ ...r, ...l } as { keyField: string; value1: number; value2: number }) 270 | ); 271 | expect(joinedArray.length).toBe(2); 272 | expect(joinedArray[1].keyField).toBe('k2'); 273 | expect(joinedArray[1].value1).toBe(22); 274 | expect(joinedArray[1].value2).toBe(32); 275 | }); 276 | 277 | it('fullJoin', () => { 278 | const arr1: { keyField: string; value1: number }[] = [ 279 | { keyField: 'k1', value1: 21 }, 280 | { keyField: 'k2', value1: 22 }, 281 | { keyField: 'k3', value1: 23 }, 282 | { keyField: 'k4', value1: 24 } 283 | ]; 284 | 285 | const arr2: { keyField: string; value2: number }[] = [ 286 | { keyField: 'k1', value2: 31 }, 287 | { keyField: 'k2', value2: 32 }, 288 | { keyField: 'k5', value2: 35 }, 289 | { keyField: 'k8', value2: 38 } 290 | ]; 291 | 292 | const joinedArray = fullJoin( 293 | arr1, 294 | arr2, 295 | l => l.keyField, 296 | r => r.keyField, 297 | (l, r) => ({ ...r, ...l } as { keyField: string; value1: number; value2: number }) 298 | ); 299 | expect(joinedArray.length).toBe(6); 300 | expect(joinedArray[1].keyField).toBe('k2'); 301 | expect(joinedArray[1].value1).toBe(22); 302 | expect(joinedArray[1].value2).toBe(32); 303 | }); 304 | 305 | it('simple pivot', () => { 306 | const arr = [ 307 | { product: 'P1', year: '2018', sale: '11' }, 308 | { product: 'P1', year: '2019', sale: '12' }, 309 | { product: 'P2', year: '2018', sale: '21' }, 310 | { product: 'P2', year: '2019', sale: '22' } 311 | ]; 312 | 313 | const res = pivot(arr, 'product', 'year', 'sale'); 314 | expect(res.length).toBe(2); 315 | expect(res.filter(r => r.product === 'P1')[0]['2018']).toBe(11); 316 | expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); 317 | }); 318 | 319 | it('pivot with default sum', () => { 320 | const arr = [ 321 | { product: 'P1', year: '2018', sale: '11' }, 322 | { product: 'P1', year: '2019', sale: '12' }, 323 | { product: 'P1', year: '2019', sale: '22' }, 324 | { product: 'P2', year: '2018', sale: '21' }, 325 | { product: 'P2', year: '2019', sale: '22' } 326 | ]; 327 | 328 | const res = pivot(arr, 'product', 'year', 'sale'); 329 | expect(res.length).toBe(2); 330 | expect(res.filter(r => r.product === 'P1')[0]['2019']).toBe(34); 331 | expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); 332 | }); 333 | 334 | it('pivot with specified AVG', () => { 335 | const arr: { product: string; year: string; sale: string }[] = [ 336 | { product: 'P1', year: '2018', sale: '11' }, 337 | { product: 'P1', year: '2019', sale: '12' }, 338 | { product: 'P1', year: '2019', sale: '22' }, 339 | { product: 'P2', year: '2018', sale: '21' }, 340 | { product: 'P2', year: '2019', sale: '22' } 341 | ]; 342 | 343 | const res = pivot(arr, 'product', 'year', 'sale', avg); 344 | expect(res.length).toBe(2); 345 | expect(res.filter(r => r.product === 'P1')[0]['2019']).toBe(17); 346 | expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); 347 | }); 348 | 349 | it('pivot with null value', () => { 350 | const arr: { product: string; year: string; sale: string }[] = [ 351 | { product: 'P1', year: '2018', sale: '11' }, 352 | { product: 'P1', year: '2019', sale: '12' }, 353 | { product: 'P1', year: '2019', sale: '22' }, 354 | { product: 'P2', year: '2018', sale: '21' }, 355 | { product: 'P2', year: '2019', sale: '22' }, 356 | { product: 'P3', year: '2019', sale: '33' } 357 | ]; 358 | 359 | const res = pivot(arr, 'product', 'year', 'sale', avg); 360 | expect(res.length).toBe(3); 361 | expect(res.filter(r => r.product === 'P1')[0]['2019']).toBe(17); 362 | expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); 363 | expect(res.filter(r => r.product === 'P3')[0]['2019']).toBe(33); 364 | expect(res.filter(r => r.product === 'P3')[0]['2018']).toBe(null); 365 | }); 366 | 367 | it('pivot with null value with sum', () => { 368 | const arr: { product: string; year: string; sale: string }[] = [ 369 | { product: 'P1', year: '2018', sale: '11' }, 370 | { product: 'P1', year: '2019', sale: '12' }, 371 | { product: 'P1', year: '2019', sale: '22' }, 372 | { product: 'P2', year: '2018', sale: '21' }, 373 | { product: 'P2', year: '2019', sale: '22' }, 374 | { product: 'P3', year: '2019', sale: '33' } 375 | ]; 376 | 377 | const res = pivot(arr, 'product', 'year', 'sale', sum); 378 | expect(res.length).toBe(3); 379 | expect(res.filter(r => r.product === 'P1')[0]['2019']).toBe(34); 380 | expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); 381 | expect(res.filter(r => r.product === 'P3')[0]['2019']).toBe(33); 382 | expect(res.filter(r => r.product === 'P3')[0]['2018']).toBe(null); 383 | }); 384 | 385 | it('pivot not string data value', () => { 386 | const arr = [ 387 | { product: 'P1', year: '2018', notAString: 'Data11' }, 388 | { product: 'P1', year: '2019', notAString: 'Data12' }, 389 | { product: 'P2', year: '2018', notAString: 'Data21' }, 390 | { product: 'P2', year: '2019', notAString: 'Data22' }, 391 | { product: 'P3', year: '2019', notAString: 'Data33' } 392 | ]; 393 | 394 | const res = pivot(arr, 'year', 'product', 'notAString', first); 395 | expect(res.length).toBe(2); 396 | // expect(res.filter(r => r.product === '2019')[0]['P1']).toBe('Data12'); 397 | // expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe('Data21'); 398 | // expect(res.filter(r => r.product === 'P3')[0]['2019']).toBe('Data33'); 399 | // expect(res.filter(r => r.product === 'P3')[0]['2018']).toBe(null); 400 | }); 401 | 402 | it('stats quantile for sorted array', () => { 403 | const numbersData = [3, 1, 2, 4, 0].sort(); 404 | expect(quantile(numbersData, 0)).toBe(0); 405 | expect(quantile(numbersData, 1 / 4)).toBe(1); 406 | expect(quantile(numbersData, 1.5 / 4)).toBe(1.5); 407 | expect(quantile(numbersData, 2 / 4)).toBe(2); 408 | expect(quantile(numbersData, 2.5 / 4)).toBe(2.5); 409 | expect(quantile(numbersData, 3 / 4)).toBe(3); 410 | expect(quantile(numbersData, 3.2 / 4)).toBe(3.2); 411 | expect(quantile(numbersData, 4 / 4)).toBe(4); 412 | 413 | const even = [3, 6, 7, 8, 8, 10, 13, 15, 16, 20]; 414 | expect(quantile(even, 0)).toBe(3); 415 | expect(quantile(even, 0.25)).toBe(7.25); 416 | expect(quantile(even, 0.5)).toBe(9); 417 | expect(quantile(even, 0.75)).toBe(14.5); 418 | expect(quantile(even, 1)).toBe(20); 419 | 420 | const odd = [3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20]; 421 | expect(quantile(odd, 0)).toBe(3); 422 | expect(quantile(odd, 0.25)).toBe(7.5); 423 | expect(quantile(odd, 0.5)).toBe(9); 424 | expect(quantile(odd, 0.75)).toBe(14); 425 | expect(quantile(odd, 1)).toBe(20); 426 | 427 | expect(quantile([3, 5, 10], 0.5)).toBe(5); 428 | }); 429 | 430 | it('stats mean', () => { 431 | expect(mean([1])).toBe(1); 432 | expect(mean([5, 1, 2, 3, 4])).toBe(3); 433 | expect(mean([19, 4])).toBe(11.5); 434 | expect(mean([4, 19])).toBe(11.5); 435 | expect(mean([NaN, 1, 2, 3, 4, 5])).toBe(3); 436 | expect(mean([1, 2, 3, 4, 5, NaN])).toBe(3); 437 | expect(mean([9, null, 4, undefined, 5, NaN])).toBe(6); 438 | }); 439 | 440 | it('stats variance', () => { 441 | expect(variance([5, 1, 2, 3, 4])).toBe(2.5); 442 | expect(variance([20, 3])).toBe(144.5); 443 | expect(variance([3, 20])).toBe(144.5); 444 | expect(variance([NaN, 1, 2, 3, 4, 5])).toBe(2.5); 445 | expect(variance([1, 2, 3, 4, 5, NaN])).toBe(2.5); 446 | expect(variance([10, null, 3, undefined, 5, NaN])).toBe(13); 447 | }); 448 | 449 | it('stats stdev', () => { 450 | expect(stdev([5, 1, 2, 3, 4])).toEqual(Math.sqrt(2.5)); 451 | expect(stdev([20, 3])).toEqual(Math.sqrt(144.5)); 452 | expect(stdev([3, 20])).toEqual(Math.sqrt(144.5)); 453 | expect(stdev([NaN, 1, 2, 3, 4, 5])).toEqual(Math.sqrt(2.5)); 454 | expect(stdev([1, 2, 3, 4, 5, NaN])).toEqual(Math.sqrt(2.5)); 455 | expect(stdev([10, null, 3, undefined, 5, NaN])).toEqual(Math.sqrt(13)); 456 | }); 457 | 458 | it('stats median', () => { 459 | expect(median([1])).toBe(1); 460 | expect(median([5, 1, 2, 3])).toBe(2.5); 461 | expect(median([5, 1, 2, 3, 4])).toBe(3); 462 | expect(median([20, 3])).toBe(11.5); 463 | expect(median([3, 20])).toBe(11.5); 464 | expect(median([10, 3, 5])).toBe(5); 465 | expect(median([NaN, 1, 2, 3, 4, 5])).toBe(3); 466 | expect(median([1, 2, 3, 4, 5, NaN])).toBe(3); 467 | expect(median([null, 3, undefined, 5, NaN, 10])).toBe(5); 468 | expect(median([10, null, 3, undefined, 5, NaN])).toBe(5); 469 | }); 470 | 471 | it('utils sort', () => { 472 | let arr = pipeFuncs.sort(data, 'country DESC', 'name ASC') || []; 473 | 474 | expect(arr[0].name).toEqual('Bill'); 475 | arr = pipeFuncs.sort(data, 'age ASC', 'name DESC'); 476 | expect(arr[2].name).toBe('Marry'); 477 | const numArr = pipeFuncs.sort([5, 2, 9, 4]); 478 | expect(numArr[2]).toBe(5); 479 | 480 | const arrWithUndefinedProps = [ 481 | { name: 'Tom', age: 7 }, 482 | { name: 'Bob', age: 10 }, 483 | { age: 5 }, 484 | { name: 'Jerry', age: 3 } 485 | ]; 486 | let objArr = pipeFuncs.sort(arrWithUndefinedProps, 'name ASC') || []; 487 | expect(objArr[0].age).toBe(5); 488 | objArr = pipeFuncs.sort(arrWithUndefinedProps, 'name DESC') || []; 489 | expect(objArr[0].age).toBe(7); 490 | }); 491 | 492 | it('toObject test', () => { 493 | const array = [ 494 | { name: 'Tom', age: 7 }, 495 | { name: 'Bob', age: 10 }, 496 | { age: 5 }, 497 | { name: 'Jerry', age: 3 } 498 | ]; 499 | 500 | const obj1 = toObject(array, 'name'); 501 | expect(Object.keys(obj1).length).toBe(array.length); 502 | expect(obj1['Bob'].age).toBe(10); 503 | expect(obj1['undefined'].age).toBe(5); 504 | 505 | const obj2 = toObject(array, i => i.name as unknown as string); 506 | expect(Object.keys(obj2).length).toBe(array.length); 507 | 508 | // make sure both are thesame 509 | expect(JSON.stringify(obj1)).toBe(JSON.stringify(obj2)); 510 | }); 511 | 512 | it('toObject NOT string', () => { 513 | const array = [ 514 | { name: 'Tom', age: 7, date: new Date(2020, 0, 8) }, 515 | { name: 'Bob', age: 10, date: new Date(2020, 0, 2) }, 516 | { age: 5, date: new Date(2020, 0, 3) }, 517 | { name: 'Jerry', age: 3, date: new Date(2020, 0, 4) } 518 | ]; 519 | 520 | expect(toObject(array, i => i.age as unknown as string)['7'].name).toBe('Tom'); 521 | expect(toObject(array, 'age')['7'].name).toBe('Tom'); 522 | 523 | expect(toObject(array, i => i.date as unknown as string)['2020-01-02'].name).toBe('Bob'); 524 | expect(toObject(array, 'date')['2020-01-02'].name).toBe('Bob'); 525 | }); 526 | 527 | it('toSeries > ', () => { 528 | const array = [ 529 | { name: 'Tom', age: 7, date: new Date(2020, 0, 8) }, 530 | { name: 'Bob', age: 10, date: new Date(2020, 0, 2) }, 531 | { age: 5, date: new Date(2020, 0, 3) }, 532 | { name: 'Jerry', age: 3, date: new Date(2020, 0, 4) } 533 | ]; 534 | 535 | expect(toSeries(array, 'name').length).toBe(4); 536 | expect((toSeries(array, 'name') as unknown[])[2]).toBe(null); 537 | expect((toSeries(array) as Record)['name'].length).toBe(4); 538 | expect((toSeries(array) as Record)['age'].length).toBe(4); 539 | expect(Object.keys(toSeries(array, ['name', 'age'])).length).toBe(2); 540 | }); 541 | }); 542 | -------------------------------------------------------------------------------- /src/utils/helpers.ts: -------------------------------------------------------------------------------- 1 | import { Selector, FieldDescription, DataTypeName, ScalarType, PrimitiveType } from '../types'; 2 | 3 | /** 4 | * Formats selected value to number. 5 | * @private 6 | * @param val Primitive or object. 7 | * @param elementSelector Function invoked per iteration. 8 | */ 9 | export function parseNumber(val: ScalarType, elementSelector?: Selector): number | undefined { 10 | if (elementSelector && typeof elementSelector === 'function') { 11 | val = elementSelector(val); 12 | } 13 | if (val instanceof Date) { 14 | return val.getTime(); 15 | } 16 | switch (typeof val) { 17 | case 'string': { 18 | const fV = parseFloat(val); 19 | if (!isNaN(fV)) { 20 | return fV; 21 | } 22 | break; 23 | } 24 | case 'boolean': 25 | return Number(val); 26 | case 'number': 27 | return isNaN(val) ? undefined : val; 28 | } 29 | } 30 | 31 | export function parseNumberOrNull(value: string | number): number | null { 32 | if (typeof value === 'number') { 33 | return value; 34 | } 35 | 36 | if (!value || typeof value !== 'string') { 37 | return null; 38 | } 39 | 40 | value = value.trim(); 41 | 42 | // Just to make sure string contains digits only and '.', ','. Otherwise, parseFloat can incorrectly parse into number 43 | for (let i = value.length - 1; i >= 0; i--) { 44 | const d = value.charCodeAt(i); 45 | if (d < 48 || d > 57) { 46 | // '.' - 46 ',' - 44 '-' - 45(but only first char) 47 | if (d !== 46 && d !== 44 && (d !== 45 || i !== 0)) return null; 48 | } 49 | } 50 | 51 | const res = parseFloat(value.replace(/,/g, '')); 52 | return !isNaN(res) ? res : null; 53 | } 54 | 55 | /** 56 | * More wider datetime parser 57 | * @param value 58 | */ 59 | export function parseDatetimeOrNull( 60 | value: string | Date | number, 61 | format: string | null = null 62 | ): Date | null { 63 | format = (format || '').toLowerCase(); 64 | if (!value) { 65 | return null; 66 | } 67 | if (value instanceof Date && !isNaN(value.valueOf())) { 68 | return value; 69 | } 70 | 71 | // only string values can be converted to Date 72 | const tpOf = typeof value; 73 | if (tpOf === 'number' || tpOf === 'bigint') { 74 | return new Date(value); 75 | } else if (tpOf !== 'string') { 76 | return null; 77 | } 78 | 79 | value = value as string; 80 | 81 | const strValue = String(value); 82 | if (!strValue.length) { 83 | return null; 84 | } 85 | 86 | const parseMonth = (mm: string): number => { 87 | if (!mm || !mm.length) { 88 | return NaN; 89 | } 90 | 91 | const m = parseInt(mm, 10); 92 | if (!isNaN(m)) { 93 | return m - 1; 94 | } 95 | 96 | // make sure english months are coming through 97 | if (mm.startsWith('jan')) { 98 | return 0; 99 | } 100 | if (mm.startsWith('feb')) { 101 | return 1; 102 | } 103 | if (mm.startsWith('mar')) { 104 | return 2; 105 | } 106 | if (mm.startsWith('apr')) { 107 | return 3; 108 | } 109 | if (mm.startsWith('may')) { 110 | return 4; 111 | } 112 | if (mm.startsWith('jun')) { 113 | return 5; 114 | } 115 | if (mm.startsWith('jul')) { 116 | return 6; 117 | } 118 | if (mm.startsWith('aug')) { 119 | return 7; 120 | } 121 | if (mm.startsWith('sep')) { 122 | return 8; 123 | } 124 | if (mm.startsWith('oct')) { 125 | return 9; 126 | } 127 | if (mm.startsWith('nov')) { 128 | return 10; 129 | } 130 | if (mm.startsWith('dec')) { 131 | return 11; 132 | } 133 | 134 | return NaN; 135 | }; 136 | 137 | const correctYear = (yy: number): number => { 138 | if (yy < 100) { 139 | return yy < 68 ? yy + 2000 : yy + 1900; 140 | } else { 141 | return yy; 142 | } 143 | }; 144 | 145 | const validDateOrNull = ( 146 | yyyy: number, 147 | month: number, 148 | day: number, 149 | hours: number, 150 | mins: number, 151 | ss: number, 152 | ms: number 153 | ): Date | null => { 154 | if (month > 11 || day > 31 || hours >= 60 || mins >= 60 || ss >= 60) { 155 | return null; 156 | } 157 | 158 | if (ms > 1000) { 159 | ms = parseInt(String(ms).substring(0, 3)); 160 | } 161 | const dd = new Date(yyyy, month, day, hours, mins, ss, ms); 162 | return !isNaN(dd.valueOf()) ? dd : null; 163 | }; 164 | 165 | const strTokens = strValue 166 | .replace('T', ' ') 167 | .replace('Z', ' ') 168 | .replace('.', ' ') 169 | .toLowerCase() 170 | .split(/[: /-]/); 171 | const dt = strTokens.map(s => parseNumberOrNull(s) ?? Number.NaN); 172 | 173 | let d: Date | null = null; 174 | 175 | if ( 176 | format.startsWith('mm/dd/yy') || 177 | format.startsWith('mmm/dd/yy') || 178 | format.startsWith('mm-dd-yy') || 179 | format.startsWith('mmm-dd-yy') 180 | ) { 181 | // handle US format 182 | return validDateOrNull( 183 | correctYear(dt[2]), 184 | parseMonth(strTokens[0]), 185 | dt[1], 186 | dt[3] || 0, 187 | dt[4] || 0, 188 | dt[5] || 0, 189 | dt[6] || 0 190 | ); 191 | } else if (format.startsWith('yyyymm')) { 192 | return validDateOrNull( 193 | parseInt(value.substring(0, 4)), 194 | parseInt(value.substring(4, 6)) - 1, 195 | value.length > 6 ? parseInt(value.substring(6, 8)) : 1, 196 | 0, 197 | 0, 198 | 0, 199 | 0 200 | ); 201 | } else if ( 202 | format.startsWith('dd/mm/yy') || 203 | format.startsWith('dd/mmm/yy') || 204 | format.startsWith('dd-mm-yy') || 205 | format.startsWith('dd-mmm-yy') 206 | ) { 207 | return validDateOrNull( 208 | correctYear(dt[2]), 209 | parseMonth(strTokens[1]), 210 | dt[0], 211 | dt[3] || 0, 212 | dt[4] || 0, 213 | dt[5] || 0, 214 | dt[6] || 0 215 | ); 216 | } else if (format.startsWith('yyyy-mm')) { 217 | return validDateOrNull( 218 | dt[0], 219 | parseMonth(strTokens[1]), 220 | dt[2] || 1, 221 | dt[3] || 0, 222 | dt[4] || 0, 223 | dt[5] || 0, 224 | dt[6] || 0 225 | ); 226 | } else if (format.length) { 227 | throw new Error(`Unrecognized format '${format}'`); 228 | } 229 | 230 | // try ISO first 231 | d = validDateOrNull(dt[0], dt[1] - 1, dt[2], dt[3] || 0, dt[4] || 0, dt[5] || 0, dt[6] || 0); 232 | if (d) { 233 | return d; 234 | } 235 | 236 | // then UK 237 | d = validDateOrNull( 238 | correctYear(dt[2]), 239 | parseMonth(strTokens[1]), 240 | dt[0], 241 | dt[3] || 0, 242 | dt[4] || 0, 243 | dt[5] || 0, 244 | dt[6] || 0 245 | ); 246 | if (d) { 247 | return d; 248 | } 249 | 250 | // we can't accept both UK and US format as it is ambiguous 251 | // for US use format parameter e.g. MM/dd/yyyy 252 | // // then US guess 253 | // return validDateOrNull( 254 | // correctYear(dt[2]), 255 | // parseMonth(strTokens[0]), 256 | // dt[1], 257 | // dt[3] || 0, 258 | // dt[4] || 0, 259 | // dt[5] || 0, 260 | // dt[6] || 0 261 | // ); 262 | 263 | return null; 264 | } 265 | 266 | export function parseBooleanOrNull(val: boolean | string): boolean | null { 267 | if (!val) { 268 | return null; 269 | } 270 | if (typeof val === 'boolean') { 271 | return val; 272 | } 273 | 274 | const trulyVals = ['1', 'yes', 'true', 'on']; 275 | const falsyVals = ['0', 'no', 'false', 'off']; 276 | const checkVal = val.toString().toLowerCase().trim(); 277 | 278 | if (trulyVals.includes(checkVal)) { 279 | return true; 280 | } 281 | 282 | if (falsyVals.includes(checkVal)) { 283 | return false; 284 | } 285 | 286 | return null; 287 | } 288 | 289 | export function addDays(dt: Date, daysOffset: number): Date { 290 | if (!dt || !(dt instanceof Date)) { 291 | throw new Error('First parameter must be Date'); 292 | } 293 | dt = new Date(dt.valueOf()); 294 | dt.setDate(dt.getDate() + daysOffset); 295 | return dt; 296 | } 297 | 298 | export function addBusinessDays(dt: Date, bDaysOffset: number, holidays?: (Date | string)[]): Date { 299 | const date = parseDatetimeOrNull(dt); 300 | const holidayDates = 301 | holidays 302 | ?.map(d => parseDatetimeOrNull(d)) 303 | ?.filter(d => !!d) 304 | ?.map(d => d?.toDateString()) || []; 305 | if (!date) { 306 | throw new Error(`A first parameter to 'addBusinessdays' must be Date`); 307 | } 308 | dt = new Date(date.valueOf()); 309 | dt.setDate(dt.getDate() + bDaysOffset); 310 | 311 | // skip saturdays and sundays 312 | if (bDaysOffset < 0) { 313 | while (dt.getDay() === 0 || dt.getDay() === 6 || holidayDates.includes(dt.toDateString())) { 314 | dt.setDate(dt.getDate() - 1); 315 | } 316 | } else { 317 | while (dt.getDay() === 0 || dt.getDay() === 6 || holidayDates.includes(dt.toDateString())) { 318 | dt.setDate(dt.getDate() + 1); 319 | } 320 | } 321 | return dt; 322 | } 323 | 324 | export function dateToString(d: Date, format?: string): string { 325 | if (!(d instanceof Date)) { 326 | throw new Error(`A first parameter to 'dateToString' must be Date`); 327 | } 328 | 329 | const date = new Date(d.valueOf()); 330 | date.setMinutes(date.getMinutes() - date.getTimezoneOffset()); 331 | const strDate = date.toISOString().replace('T00:00:00.000Z', ''); 332 | 333 | if (format) { 334 | // a quick and dirty way to achive a most used formats 335 | const t = strDate.split(/[.T:Z /-]/); 336 | const f = { 337 | yyyy: t[0], 338 | yy: t[0].slice(2), 339 | mm: t[1], 340 | dd: t[2] 341 | }; 342 | 343 | if (format.toLowerCase() === 'dd/mm/yyyy') { 344 | return `${f.dd}/${f.mm}/${f.yyyy}`; 345 | } else if (format.toLowerCase() === 'mm/dd/yyyy') { 346 | return `${f.mm}/${f.dd}/${f.yyyy}`; 347 | } else if (format.toLowerCase() === 'dd/mm/yy') { 348 | return `${f.dd}/${f.mm}/${f.yy}`; 349 | } else if (format.toLowerCase() === 'yyyymmdd') { 350 | return `${f.yyyy}${f.mm}${f.dd}`; 351 | } else if (format.toLowerCase() === 'mm-dd-yyyy') { 352 | return `${f.mm}-${f.dd}-${f.yyyy}`; 353 | } else if (format.toLowerCase() === 'mm-dd-yy') { 354 | return `${f.mm}-${f.dd}-${f.yy}`; 355 | } else if (format.toLowerCase() === 'dd-mm-yyyy') { 356 | return `${f.dd}-${f.mm}-${f.yyyy}`; 357 | } else if (format.toLowerCase() === 'yyyy-mm-dd') { 358 | return `${f.yyyy}-${f.mm}-${f.dd}`; 359 | } else { 360 | throw new Error(`Unsupported format ${format}`); 361 | } 362 | } 363 | 364 | return strDate; 365 | } 366 | 367 | export function deepClone(obj: T): T { 368 | if (obj == null || obj == undefined) { 369 | return obj; 370 | } 371 | 372 | if (obj instanceof Date) { 373 | return new Date(obj.getTime()) as unknown as T; 374 | } 375 | 376 | if (Array.isArray(obj)) { 377 | return obj.map(v => deepClone(v)) as unknown as T; 378 | } 379 | 380 | if (typeof obj === 'object') { 381 | const clone = {} as Record; 382 | for (const propName in obj) { 383 | const propValue = (obj as Record)[propName]; 384 | 385 | if (propValue == null || propValue == undefined) { 386 | clone[propName] = propValue; 387 | } else if (propValue instanceof Date) { 388 | clone[propName] = new Date(propValue.getTime()); 389 | } else if (Array.isArray(propValue)) { 390 | clone[propName] = propValue.map(v => deepClone(v)); 391 | } else if (typeof propValue == 'object') { 392 | clone[propName] = deepClone(propValue); 393 | } else { 394 | clone[propName] = propValue; 395 | } 396 | } 397 | return clone as T; 398 | } 399 | 400 | return obj; 401 | } 402 | 403 | export function workoutDataType( 404 | value: ScalarType, 405 | inType: DataTypeName | undefined 406 | ): DataTypeName | undefined { 407 | function getRealType(val: ScalarType): DataTypeName { 408 | function processNumber(num: number): DataTypeName { 409 | if (num % 1 === 0) { 410 | return num > 2147483647 ? DataTypeName.BigIntNumber : DataTypeName.WholeNumber; 411 | } else { 412 | return DataTypeName.FloatNumber; 413 | } 414 | } 415 | 416 | let num = null; 417 | let bl = null; 418 | let dt = null; 419 | 420 | switch (typeof val) { 421 | case 'boolean': 422 | return DataTypeName.Boolean; 423 | case 'number': 424 | return processNumber(val); 425 | case 'object': 426 | if (val instanceof Date) { 427 | const dt = val; 428 | return dt.getHours() === 0 && dt.getMinutes() === 0 && dt.getSeconds() === 0 429 | ? DataTypeName.Date 430 | : DataTypeName.DateTime; 431 | } 432 | 433 | return DataTypeName.String; 434 | case 'string': 435 | dt = parseDatetimeOrNull(val); 436 | if (dt) { 437 | return dt.getHours() === 0 && dt.getMinutes() === 0 && dt.getSeconds() === 0 438 | ? DataTypeName.Date 439 | : DataTypeName.DateTime; 440 | } 441 | 442 | num = parseNumberOrNull(val); 443 | if (num !== null) { 444 | return processNumber(num); 445 | } 446 | 447 | bl = parseBooleanOrNull(val); 448 | if (bl !== null) { 449 | return DataTypeName.Boolean; 450 | } 451 | 452 | return val.length > 4000 ? DataTypeName.LargeString : DataTypeName.String; 453 | 454 | default: 455 | return DataTypeName.LargeString; 456 | } 457 | } 458 | 459 | if (value == null || value == undefined) { 460 | return undefined; 461 | } 462 | 463 | // no point to proceed, string is most common type 464 | if (inType === DataTypeName.LargeString) { 465 | return DataTypeName.LargeString; 466 | } 467 | 468 | const realType = getRealType(value); 469 | 470 | if (inType === undefined) { 471 | return realType; 472 | } else { 473 | // normal case. Means all values in column are the same 474 | if (inType === realType) { 475 | return inType; 476 | } 477 | 478 | // date / datetime case 479 | if ( 480 | (inType === DataTypeName.Date && realType === DataTypeName.DateTime) || 481 | (inType === DataTypeName.DateTime && realType === DataTypeName.Date) 482 | ) { 483 | return DataTypeName.DateTime; 484 | } 485 | 486 | // if any of items are string, then it must be string 487 | if (realType === DataTypeName.String) { 488 | return DataTypeName.String; 489 | } 490 | if (inType === DataTypeName.String && realType !== DataTypeName.LargeString) { 491 | return DataTypeName.String; 492 | } 493 | 494 | if (inType === DataTypeName.FloatNumber) { 495 | return DataTypeName.FloatNumber; 496 | } 497 | if (realType === DataTypeName.FloatNumber && inType === DataTypeName.WholeNumber) { 498 | return DataTypeName.FloatNumber; 499 | } 500 | 501 | if (realType === DataTypeName.BigIntNumber) { 502 | return DataTypeName.BigIntNumber; 503 | } 504 | if (inType === DataTypeName.BigIntNumber && realType === DataTypeName.WholeNumber) { 505 | return DataTypeName.BigIntNumber; 506 | } 507 | 508 | if (realType !== inType && realType !== DataTypeName.LargeString) { 509 | return DataTypeName.String; 510 | } 511 | 512 | return DataTypeName.LargeString; 513 | } 514 | 515 | return undefined; 516 | } 517 | 518 | /** 519 | * generates a field descriptions (first level only) that can be used for relational table definition. 520 | * if any properties are Objects, it would use JSON.stringify to calculate maxSize field. 521 | * @param items 522 | */ 523 | export function getFieldsInfo( 524 | items: Record[] | ScalarType[] 525 | ): FieldDescription[] { 526 | const resultMap: Record = Object.create(null); 527 | const valuesMap: Record> = Object.create(null); 528 | let index = 0; 529 | 530 | function processItem( 531 | name: string, 532 | value: string | number | bigint | boolean | Date | null 533 | ): void { 534 | let fDesc = resultMap[name]; 535 | let valuesSet = valuesMap[name]; 536 | 537 | if (valuesSet === undefined) { 538 | valuesSet = valuesMap[name] = new Set(); 539 | } 540 | 541 | if (fDesc === undefined) { 542 | fDesc = { 543 | index: index++, 544 | fieldName: name, 545 | isNullable: false, 546 | isObject: false 547 | } as FieldDescription; 548 | resultMap[name] = fDesc; 549 | } 550 | 551 | let strValue: PrimitiveType | null = null; 552 | if (value === null || value === undefined) { 553 | fDesc.isNullable = true; 554 | } else { 555 | strValue = 556 | value instanceof Date 557 | ? dateToString(value) 558 | : typeof value === 'object' 559 | ? JSON.stringify(value) 560 | : String(value); 561 | 562 | if (!fDesc.isObject && !(value instanceof Date)) { 563 | fDesc.isObject = typeof value === 'object'; 564 | } 565 | 566 | const newType = workoutDataType(value, fDesc.dataTypeName); 567 | 568 | if ( 569 | newType !== fDesc.dataTypeName && 570 | // special case when datetime can't be date again 571 | !(fDesc.dataTypeName === DataTypeName.DateTime && newType === DataTypeName.Date) 572 | ) { 573 | fDesc.dataTypeName = newType; 574 | } 575 | 576 | if ( 577 | (fDesc.dataTypeName == DataTypeName.String || 578 | fDesc.dataTypeName == DataTypeName.LargeString) && 579 | strValue.length > (fDesc.maxSize || 0) 580 | ) { 581 | fDesc.maxSize = strValue.length; 582 | } 583 | } 584 | 585 | if (!valuesSet.has(strValue)) { 586 | valuesSet.add(strValue); 587 | } 588 | } 589 | 590 | for (let i = 0; i < items.length; i++) { 591 | const item = items[i]; 592 | if ( 593 | item !== null && 594 | item !== undefined && 595 | typeof item === 'object' && 596 | !(item instanceof Date) && 597 | !Array.isArray(item) 598 | ) { 599 | for (const [name, value] of Object.entries(item as Record)) { 600 | processItem(name, value); 601 | } 602 | } else { 603 | processItem('_value_', item as ScalarType); 604 | } 605 | } 606 | 607 | const fields = Object.values(resultMap).sort((a: FieldDescription, b: FieldDescription) => 608 | a.index > b.index ? 1 : b.index > a.index ? -1 : 0 609 | ); 610 | 611 | return fields.map(r => { 612 | r.isUnique = valuesMap[r.fieldName].size === items.length; 613 | return r; 614 | }); 615 | } 616 | 617 | export function processJson( 618 | jsonString: string, 619 | handlePathFunc: (path: (string | number)[], row: number, col: number) => boolean 620 | ): void { 621 | if (typeof handlePathFunc !== 'function') { 622 | throw new Error('handlePathFunc is not provided.'); 623 | } 624 | 625 | function tokenize( 626 | json: string, 627 | tokenFunc: (token: string, row: number, column: number) => boolean 628 | ): void { 629 | let line = 1, 630 | column = 0; 631 | 632 | let isString = false; 633 | let currentString = ''; 634 | let previousToken = ''; 635 | 636 | for (let i = 0; i < json.length; i++) { 637 | column++; 638 | const currChar = json[i]; 639 | if (currChar === '\n') { 640 | line++; 641 | column = 0; 642 | continue; 643 | } 644 | 645 | if (currChar === '"' && json[i - 1] !== '\\') { 646 | isString = !isString; 647 | // skip string value and empty strings 648 | if (currentString && previousToken !== ':') { 649 | if (typeof tokenFunc === 'function' && tokenFunc(currentString, line, column)) { 650 | break; 651 | } 652 | previousToken = currentString; 653 | } 654 | currentString = ''; 655 | continue; 656 | } 657 | 658 | if (isString) { 659 | currentString += currChar; 660 | } else if (['[', ']', ':', '{', '}', ','].indexOf(currChar) >= 0) { 661 | if (typeof tokenFunc === 'function' && tokenFunc(currChar, line, column)) { 662 | break; 663 | } 664 | previousToken = currentString; 665 | } 666 | } 667 | } 668 | 669 | const currentPath: (string | number)[] = []; 670 | let isNextTokenAKey = false; 671 | 672 | function handleToken(token: string, row: number, column: number): boolean { 673 | const isArrayLevel: () => boolean = () => 674 | typeof currentPath[currentPath.length - 1] === 'number'; 675 | 676 | if (token === '{') { 677 | if (isArrayLevel()) { 678 | (currentPath[currentPath.length - 1] as number)++; 679 | if (handlePathFunc([...currentPath], row, column)) { 680 | return true; 681 | } 682 | } 683 | 684 | isNextTokenAKey = true; 685 | 686 | return false; 687 | } 688 | if (token === '}') { 689 | if (!isNextTokenAKey) { 690 | currentPath.pop(); 691 | } 692 | isNextTokenAKey = false; 693 | return false; 694 | } 695 | 696 | if (token === ',') { 697 | if (!isArrayLevel()) { 698 | isNextTokenAKey = true; 699 | currentPath.pop(); 700 | } 701 | return false; 702 | } 703 | 704 | if (token === '[') { 705 | currentPath.push(-1); 706 | return false; 707 | } 708 | 709 | if (token === ']') { 710 | currentPath.pop(); 711 | return false; 712 | } 713 | 714 | if (isNextTokenAKey) { 715 | isNextTokenAKey = false; 716 | currentPath.push(token); 717 | if (handlePathFunc([...currentPath], row, column)) { 718 | return true; 719 | } 720 | } 721 | 722 | return false; 723 | } 724 | 725 | tokenize(jsonString, handleToken); 726 | } 727 | --------------------------------------------------------------------------------