├── .eslintrc.js ├── .github └── workflows │ └── test.yml ├── .gitignore ├── .prettierrc.json ├── README.md ├── arquero-playground.js ├── demo.js ├── package.json ├── rollup.config.js ├── src ├── databases │ ├── arquero │ │ ├── aq-database.js │ │ ├── aq-table-view.js │ │ └── index.js │ ├── database.js │ ├── index.js │ ├── postgres │ │ ├── index.js │ │ ├── pg-code-gen.js │ │ ├── pg-database.js │ │ ├── pg-optimizer.js │ │ ├── pg-table-view.js │ │ ├── utils │ │ │ ├── compose-queries.js │ │ │ ├── create-column.js │ │ │ └── is-reserved.js │ │ ├── verbs │ │ │ ├── common.js │ │ │ ├── concat.js │ │ │ ├── dedupe.js │ │ │ ├── derive.js │ │ │ ├── except.js │ │ │ ├── filter.js │ │ │ ├── fold.js │ │ │ ├── groupby.js │ │ │ ├── index.js │ │ │ ├── intersect.js │ │ │ ├── join.js │ │ │ ├── orderby.js │ │ │ ├── pivot.js │ │ │ ├── rollup.js │ │ │ ├── sample.js │ │ │ ├── select.js │ │ │ ├── ungroup.js │ │ │ ├── union.js │ │ │ └── unorder.js │ │ └── visitors │ │ │ ├── aggregated-columns.js │ │ │ ├── columns.js │ │ │ ├── gen-expr.js │ │ │ └── has-function.js │ └── table-view.js ├── db-table.js ├── index-node.js └── index.js ├── test ├── code-gen-test.js ├── databases │ └── postgres-test.js ├── optimizer-test.js ├── pg-utils.js ├── sql-query-test.js ├── sql-test │ └── buildSql.sql ├── tape-wrapper.js ├── to-sql-test.js ├── utils-test.js ├── utils.js ├── verbs │ ├── common.js │ ├── count-test.js │ ├── derive-test.js │ ├── filter-test.js │ ├── groupby-test.js │ ├── join-test.js │ ├── orderby-test.js │ ├── rollup-test.js │ ├── select-test.js │ ├── set-verbs-test.js │ ├── ungroup-test.js │ └── unorder-test.js └── visitors │ ├── gen-expr-test.js │ └── has-function-test.js └── yarn.lock /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: [ 3 | 'eslint:recommended' 4 | ], 5 | env: { 6 | es6: true, 7 | browser: true, 8 | node: true 9 | }, 10 | parserOptions: { 11 | ecmaVersion: 2020, 12 | sourceType: 'module' 13 | }, 14 | rules: { 15 | 'no-console': ['warn', {allow: ['warn', 'error']}], 16 | 'no-cond-assign': 'off', 17 | 'no-fallthrough': ['error', { commentPattern: 'break omitted' }], 18 | 'semi': 'error', 19 | 'quotes': ['error', 'single', { avoidEscape: true }], 20 | 'prefer-const': 'error', 21 | 'sort-imports': ['error', { 22 | ignoreCase: false, 23 | ignoreDeclarationSort: true 24 | }] 25 | } 26 | }; 27 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | 13 | strategy: 14 | matrix: 15 | node: [14, 16, 17] 16 | 17 | name: Node ${{ matrix.node }} 18 | 19 | services: 20 | postgres: 21 | image: postgres:latest 22 | env: 23 | POSTGRES_PASSWORD: password 24 | ports: 25 | - 54${{ matrix.node }}:5432 26 | options: >- 27 | --health-cmd pg_isready 28 | --health-interval 10s 29 | --health-timeout 5s 30 | --health-retries 5 31 | 32 | steps: 33 | - uses: actions/checkout@v2 34 | 35 | - name: Setup Node ${{ matrix.node }} 36 | uses: actions/setup-node@v1 37 | with: 38 | node-version: ${{ matrix.node }} 39 | 40 | - name: Install dependencies 41 | run: yarn --frozen-lockfile 42 | 43 | - name: Run tests 44 | run: yarn test 45 | env: 46 | PGUSER: postgres 47 | PGHOST: localhost 48 | PGDB: postgres 49 | PGPASSWORD: password 50 | PGPORT: 54${{ matrix.node }} 51 | 52 | - name: Run linter 53 | run: yarn lint -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | .vscode/ -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "overrides": [ 3 | { 4 | "files": "*", 5 | "options": { 6 | "printWidth": 120, 7 | "singleQuote": true, 8 | "bracketSpacing": false, 9 | "arrowParens": "avoid", 10 | "trailingComma": "all", 11 | "proseWrap": "never" 12 | } 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Arquero-SQL 2 | [Arquero](https://github.com/uwdata/arquero) is a query processing JavaScript library. 3 | So, your local machine memory is its limitation when dealing with large amount of data. 4 | Arquero-SQL is a SQL backend support for Arquero. 5 | Using the same syntax as Arquero in JavaScript, Aquero-SQL use your SQL server as an execution engine for transforming your data table. 6 | With Arquero-SQL, your data is stored in a disk and can be transformed with a powerful remote machine, suited for big data analysis tasks. 7 | 8 | Demo video: https://youtu.be/RO0luOvpiOY 9 | 10 | ## Setup 11 | ### Prerequisite 12 | Set up at PostgreSQL server. We recommend using [Docker](https://www.docker.com/) with the official PostgreSQL [image](https://hub.docker.com/_/postgres/). 13 | 14 | ```sh 15 | yarn 16 | yarn build 17 | 18 | # OR 19 | 20 | npm install 21 | npm run build 22 | ``` 23 | 24 | ## Test 25 | ### Prerequisite 26 | Set the following environment variables to your PostgreSQL server's credential 27 | - `PGDB`: Database 28 | - `PGUSER`: User name 29 | - `PGPASSWORD`: Password 30 | - `PGHOST`: Host name 31 | - `PGPORT`: Port 32 | 33 | ```sh 34 | yarn test 35 | 36 | # OR 37 | 38 | npm test 39 | ``` 40 | 41 | ## Usage 42 | ```js 43 | import * as aq from 'arquero'; 44 | import * as fs from 'fs'; 45 | import {db} from 'arquero-sql'; 46 | 47 | // Connect to a database 48 | const pg = new db.Postgres({ 49 | name, 50 | password, 51 | host, 52 | database, 53 | port 54 | }); 55 | 56 | // Create a data table 57 | const dt1 = pg.fromArquero(aq.table(...)); 58 | // OR 59 | const dt1 = pg.fromCSV(fs.readFileSync('path-to-file.csv')); 60 | 61 | // Transform the data table 62 | const dt2 = dt1 63 | .filter(d => d.Seattle > 200) 64 | .derive({new_col: d => d.Seattle + 10}); 65 | 66 | // Observe the transformation result 67 | const output = await dt2.objects(); 68 | // OR 69 | dt2.print(); 70 | ``` -------------------------------------------------------------------------------- /arquero-playground.js: -------------------------------------------------------------------------------- 1 | const {table, internal: {Query, Verbs}, op, not, desc, all, range, matches, startswith, endswith} = require('arquero'); 2 | const {toSql} = require('./dist/arquero-sql'); 3 | const {SqlQuery} = require('./dist/arquero-sql'); 4 | 5 | const dt = table({ 6 | 'Seattle': [69,108,178,207,253,268,312,281,221,142,72,52], 7 | 'Chicago': [135,136,187,215,281,311,318,283,226,193,113,106], 8 | 'San Francisco': [165,182,251,281,314,330,300,272,267,243,189,156] 9 | }); 10 | 11 | // dt.derive({d: d => op.mean(d.Chicago)}).print() 12 | 13 | dt 14 | .filter(d => op.mean(d.Chicago) > 140 || d.Seattle > 100) // is not allowed in SQL 15 | .groupby({key: d => d.Seattle > 100, S: d => d.Seattle}, 'Seattle') 16 | // .derive({k: d => d.Chicago + 10}) 17 | // .filter(d => op.mean(d.Chicago) > 140) // should becomes "having" 18 | // .filter(d => d.Chicago > 200) // tricky case -> should becomes "where" 19 | // .select('key') 20 | .rollup({key: d => op.mean(d.Chicago) + 1000}) 21 | // .select('Seattle') 22 | // .select(not('Seattle'), 'Seattle') 23 | .print(); 24 | 25 | // dt.print() 26 | dt 27 | .filter(d => d.Seattle > 100) 28 | .orderby('Chicago') 29 | // .select('Seattle') 30 | .derive({ Chicago: d => 2, Seattle: d => 1}) 31 | // .lookup(dt.derive({Seattle1: d => d.Seattle + 1000}), ['Chicago', 'Chicago'], ['Seattle1']) 32 | // .select(not(not('Seattle')), 'Chicago', 'Seattle') 33 | // .dedupe(not('Seattle', 'San Francisco')) 34 | // .select(not('Seattle'), 'Seattle', {Seattle: 'Seattle2'}, 'Chicago') 35 | // .select('Seattle', 'Chicago', {Seattle: 'Seattle1'}, 'Seattle', {Seattle: 'Seattle2'}) 36 | .groupby(not('Seattle'), {k: d => d.Seattle + d.Chicago}) 37 | .count() 38 | // .print() 39 | 40 | dt 41 | .select('Seattle', 'Chicago') 42 | .join(dt.select('Chicago', 'San Francisco'), ['Chicago', 'Chicago'], null, {suffix: ['_1', '2']}) 43 | .print() 44 | 45 | 46 | dt.groupby(['Seattle']) 47 | .rollup({a: d => op.min(d.Chicago)}) 48 | .print() 49 | // console.log(JSON.stringify( Verbs.filter(d => d.test).criteria, null, 2)); 50 | // // console.log(JSON.stringify(Verbs.derive({a: 'd => d.test'}).toAST(), null, 2)) 51 | // console.log(JSON.stringify(Verbs.groupby([{a: 'd => d.test', h: d => d.j}, 'fd', 'd', d => d.test2 + 3, {a: d => d.tt}]).toAST(), null, 2)) 52 | 53 | dt.orderby('Seattle').groupby([{g: d => ~~(d.Seattle / 100)}, {g: d => ~~(d.Seattle / 70)}]) 54 | .select('Chicago') 55 | .derive({a: d => op.min(d.Chicago), b: d => op.rank(), c: () => op.row_number(), g: 10}) 56 | .rollup() 57 | .print({limit: 20}) 58 | 59 | dt.groupby({a: d => d.Chicago > 100}) 60 | .select({Seattle: 'Chicago'}) 61 | .rollup({g: () => op.count()}) 62 | .print({limit: 20}) 63 | 64 | dt.join(dt, [d => d.Seattle, 'Seattle']) 65 | .print() 66 | 67 | dt.derive({k: `d => d['Seattle']`}) 68 | .print() 69 | 70 | console.log(dt.data().Seattle.data); 71 | 72 | 73 | for (let i = 0; i < 12; i++) { 74 | console.log(`insert into base values (${ 75 | dt.data().Seattle.data[i]}, ${ 76 | dt.data().Chicago.data[i]}, ${ 77 | dt.data()['San Francisco'].data[i]});`); 78 | } 79 | 80 | // console.log(Verbs.select('d', 'ddd', all()).toAST()) 81 | 82 | console.log(JSON.parse(dt.filter(d => d.Seattle > 200).toJSON()).data); 83 | dt.join(dt, (a, b) => a.Seattle === (b.Seattle)).print(); 84 | console.log(dt.select({Seattle: 'Seattle2'}).objects()) 85 | 86 | function dd(d) { 87 | return d.Seattle * d.Chicago 88 | } 89 | 90 | const qb = new Query(); 91 | 92 | const out = qb 93 | .derive({ 94 | a: d => d.Seattle + -d.Chicago, 95 | b: d => d.Seattle + d.Chicago, 96 | c: d => d.Chicago, 97 | d: _ => _, 98 | e: d => (d.Seattle > 10 ? 'hi' : 'test') === 'hi', 99 | // f: d => { // will not support 100 | // let a = 10; 101 | // const b = 20; 102 | // return d.Seattle > a || d.Chicago > b; 103 | // }, 104 | g: () => op.row_number(), 105 | h: d => d['Seattle'] > 10, 106 | i: d => `${d.Seattle} + ${d.Chicago}`, 107 | }) 108 | .groupby(['a', {k: d => d.s + d.e}]) 109 | .sample(d => op.mean(d.Seattle)) 110 | .select(all()); 111 | 112 | 113 | // console.log(JSON.stringify(out._verbs, null, 3)); 114 | // console.log(out._verbs[out._verbs.length - 1]); 115 | // // console.log(JSON.stringify(fromQuery(out, null), null, 3)); 116 | // console.log(JSON.stringify(out.toAST(), null, 3)); 117 | 118 | // console.log(JSON.stringify(out.toAST().verbs.map(v => { 119 | // // v.values.map(vv => toSql(vv)) 120 | // return { 121 | // verb: v.verb, 122 | // ...Object.keys(v) 123 | // .filter(k => k !== 'type' && k !== 'verb') 124 | // .reduce((acc, k) => ({ 125 | // ...acc, 126 | // [k]: Array.isArray(v[k]) 127 | // ? v[k].map(vv => toSql(vv)) 128 | // : toSql(v[k]) 129 | // }), {}) 130 | // } 131 | // }), null, 2)); 132 | 133 | // const out2 = qb 134 | // .filter(d => d.Seattle > 100) 135 | // .groupby('Seattle') 136 | // .rollup({max_Chicago: d => op.max(d.Chicago)}) 137 | // .orderby(desc(d => d.Seattle)) 138 | // .join((new TableView("test")), (a, b) => op.equal(a.Seattle, b.Chicago), ['test1']) 139 | // console.log(JSON.stringify(out.toAST(), null, 2)); 140 | -------------------------------------------------------------------------------- /demo.js: -------------------------------------------------------------------------------- 1 | const { all, desc, op, table } = require('arquero'); 2 | const {SqlQuery} = require('./dist/arquero-sql'); 3 | 4 | // Average hours of sunshine per month, from https://usclimatedata.com/. 5 | const dt = table({ 6 | 'Seattle': [69,108,178,207,253,268,312,281,221,142,72,52], 7 | 'Chicago': [135,136,187,215,281,311,318,283,226,193,113,106], 8 | 'San Francisco': [165,182,251,281,314,330,300,272,267,243,189,156] 9 | }); 10 | // Arquero-SQL 11 | const q = new SqlQuery('table', {}, {columns: ['Seattle', 'Chicago', 'San Francisco']}); 12 | 13 | // Sorted differences between Seattle and Chicago. 14 | // Table expressions use arrow function syntax. 15 | dt.derive({ 16 | month: d => op.row_number(), 17 | diff: d => d.Seattle - d.Chicago 18 | }) 19 | .select('month', 'diff') 20 | .orderby(desc('diff')) 21 | .print(); 22 | // Arquero-SQL 23 | const q1 = q.derive({ 24 | month: d => op.row_number(), 25 | diff: d => d.Seattle - d.Chicago 26 | }) 27 | .select('month', 'diff') 28 | .orderby(desc('diff')) 29 | .toSql(); 30 | console.log(q1); 31 | 32 | // Is Seattle more correlated with San Francisco or Chicago? 33 | // Operations accept column name strings outside a function context. 34 | dt.rollup({ 35 | corr_sf: op.corr('Seattle', 'San Francisco'), 36 | corr_chi: op.corr('Seattle', 'Chicago') 37 | }) 38 | .print(); 39 | // Arquero-SQL 40 | const q2 = q.rollup({ 41 | corr_sf: op.corr('Seattle', 'San Francisco'), 42 | corr_chi: op.corr('Seattle', 'Chicago') 43 | }) 44 | .toSql(); 45 | console.log(q2); 46 | 47 | // Aggregate statistics per city, as output objects. 48 | // Reshape (fold) the data to a two column layout: city, sun. 49 | dt.fold(all(), { as: ['city', 'sun'] }) 50 | .groupby('city') 51 | .rollup({ 52 | min: d => op.min(d.sun), // functional form of op.min('sun') 53 | max: d => op.max(d.sun), 54 | avg: d => op.average(d.sun), 55 | med: d => op.median(d.sun), 56 | // functional forms permit flexible table expressions 57 | skew: ({sun: s}) => (op.mean(s) - op.median(s)) / op.stdev(s) || 0 58 | }) 59 | .objects() 60 | // Arquero-SQL does not support fold 61 | const q3 = q.groupby({s: d => d.Seattle % 10}) 62 | .rollup({ 63 | min: d => op.min(d.Chicago), // functional form of op.min('sun') 64 | max: d => op.max(d.Chicago), 65 | avg: d => op.average(d.Chicago), 66 | // TODO: PostgresQL does not support median, so we need to define or do some desugaring 67 | // or extract meadian from joining other table: https://stackoverflow.com/questions/39683330/percentile-calculation-with-a-window-function 68 | med: d => op.median(d.Chicago), 69 | // functional forms permit flexible table expressions 70 | skew: ({Chicago: s}) => (op.mean(s) - op.median(s)) / op.stdev(s) || 0 71 | }) 72 | .toSql(); 73 | console.log(q3); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "arquero-sql", 3 | "version": "1.0.0", 4 | "description": "A translator for Arquero queries to SQLs", 5 | "main": "index.js", 6 | "contributors": [ 7 | "Chanwut (Mick) Kittivorawong ", 8 | "Yiming (Mike) Huang " 9 | ], 10 | "license": "MIT", 11 | "scripts": { 12 | "prebuild": "rimraf dist && mkdir dist", 13 | "build": "rollup -c", 14 | "test": "TZ=America/Los_Angeles tape 'test/**/*-test.js' --require esm", 15 | "lint": "yarn eslint src test --ext .js", 16 | "prettieresm": "cp node_modules/esm/esm.js . && prettier 'esm.js' --write && mv esm.js node_modules/esm/esm.js", 17 | "format": "yarn lint --fix && prettier '{src,test}/**/*.js' --write" 18 | }, 19 | "dependencies": { 20 | "arquero": "^3.0.0", 21 | "fast-csv": "^4.3.6", 22 | "pg": "^8.5.1", 23 | "uuid": "^8.3.2" 24 | }, 25 | "devDependencies": { 26 | "@rollup/plugin-json": "^4.1.0", 27 | "@rollup/plugin-node-resolve": "^13.3.0", 28 | "eslint": "^8.21.0", 29 | "esm": "^3.2.25", 30 | "mkdirp": "^1.0.4", 31 | "prettier": "^2.2.0", 32 | "rimraf": "^3.0.2", 33 | "rollup": "^2.77.3", 34 | "rollup-plugin-bundle-size": "1.0.3", 35 | "rollup-plugin-terser": "^7.0.2", 36 | "tape": "^5.5.3" 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import json from '@rollup/plugin-json'; 2 | import bundleSize from 'rollup-plugin-bundle-size'; 3 | import { nodeResolve } from '@rollup/plugin-node-resolve'; 4 | import { terser } from 'rollup-plugin-terser'; 5 | 6 | function onwarn(warning, defaultHandler) { 7 | if (warning.code !== 'CIRCULAR_DEPENDENCY') { 8 | defaultHandler(warning); 9 | } 10 | } 11 | 12 | const name = 'aqsql'; 13 | const external = [ 'arquero', 'fast-csv', 'fs', 'pg', 'uuid' ]; 14 | const globals = { 'arquero': 'arquero', 'fast-csv': 'csv', 'fs': 'fs', 'pg': 'pg', 'uuid': 'uuid' }; 15 | 16 | const plugins = [ 17 | json(), 18 | bundleSize(), 19 | nodeResolve({ modulesOnly: true }), 20 | ]; 21 | 22 | export default [ 23 | { 24 | input: 'src/index-node.js', 25 | external: external, 26 | plugins, 27 | onwarn, 28 | output: [ 29 | { 30 | file: 'dist/arquero-sql.node.js', 31 | format: 'cjs', 32 | name 33 | } 34 | ] 35 | }, 36 | { 37 | input: 'src/index.js', 38 | external, 39 | plugins, 40 | onwarn, 41 | output: [ 42 | { 43 | file: 'dist/arquero-sql.js', 44 | format: 'umd', 45 | globals, 46 | name 47 | }, 48 | { 49 | file: 'dist/arquero-sql.min.js', 50 | format: 'umd', 51 | sourcemap: true, 52 | plugins: [ terser({ ecma: 2018 }) ], 53 | globals, 54 | name 55 | } 56 | ] 57 | } 58 | ]; 59 | 60 | // export default { 61 | // input: 'src/index.js', 62 | // plugins: [ 63 | // json(), 64 | // bundleSize(), 65 | // nodeResolve({ modulesOnly: true }) 66 | // ], 67 | // onwarn, 68 | // output: [ 69 | // { 70 | // file: 'dist/arquero-sql.js', 71 | // name: 'aq', 72 | // format: 'umd' 73 | // }, 74 | // ] 75 | // }; -------------------------------------------------------------------------------- /src/databases/arquero/aq-database.js: -------------------------------------------------------------------------------- 1 | import * as aq from 'arquero'; 2 | import {v4 as uuid} from 'uuid'; 3 | import * as fs from 'fs'; 4 | import {DBTable} from '../../db-table'; 5 | import {Database} from '../database'; 6 | import {ArqueroTableView} from './aq-table-view'; 7 | 8 | export class ArqueroDatabase extends Database { 9 | constructor() { 10 | super(); 11 | 12 | /** @type {Map} */ 13 | this.tables = new Map(); 14 | } 15 | 16 | /** 17 | * @param {string} name 18 | */ 19 | table(name) { 20 | const pbuilder = Promise.resolve(new ArqueroTableView(this.tables.get(name), this)); 21 | return new DBTable(pbuilder); 22 | } 23 | 24 | /** 25 | * @param {string} path 26 | * @param {{name: string, type: PGType}[]} schema 27 | * @param {string} [name] 28 | */ 29 | fromCSV(path, schema, name) { 30 | name = name || `__aq__table__${uuid().split('-').join('')}__`; 31 | const result = Promise.resolve().then(() => { 32 | const table = aq.fromCSV(fs.readFileSync(path)); 33 | this.tables.set(name, table); 34 | return table; 35 | }); 36 | 37 | return new DBTable(result.then(table => new ArqueroTableView(table, this))); 38 | } 39 | 40 | /** 41 | * @param {import('arquero').internal.Table} table 42 | * @param {string} [name] 43 | * @returns {DBTable} 44 | */ 45 | fromArquero(table, name) { 46 | this.tables.set(name, table); 47 | return new DBTable(Promise.resolve(new ArqueroTableView(table, this))); 48 | } 49 | 50 | async close() {} 51 | } 52 | -------------------------------------------------------------------------------- /src/databases/arquero/aq-table-view.js: -------------------------------------------------------------------------------- 1 | import aqVerbs from 'arquero/src/verbs'; 2 | import {TableView} from '../table-view'; 3 | 4 | export class ArqueroTableView extends TableView { 5 | /** 6 | * @param {import('arquero').internal.Table} table 7 | */ 8 | constructor(table) { 9 | super(); 10 | 11 | /** @type {import('arquero').internal.Table} */ 12 | this.table = table; 13 | } 14 | 15 | /** 16 | * @param {import('arquero/src/table/table').ObjectsOptions} [options] 17 | */ 18 | async objects(options = {}) { 19 | return this.table.objects(options); 20 | } 21 | } 22 | 23 | Object.keys(aqVerbs).forEach( 24 | verb => (ArqueroTableView.prototype[verb] = (qb, ...params) => new ArqueroTableView(qb.table[verb](qb, ...params))), 25 | ); 26 | -------------------------------------------------------------------------------- /src/databases/arquero/index.js: -------------------------------------------------------------------------------- 1 | export {ArqueroDatabase} from './aq-database'; 2 | export {ArqueroTableView} from './aq-table-view'; -------------------------------------------------------------------------------- /src/databases/database.js: -------------------------------------------------------------------------------- 1 | export class Database { 2 | constructor() { 3 | if (!Database.databases) { 4 | Database.databases = []; 5 | } 6 | Database.databases.push(this); 7 | } 8 | 9 | /** 10 | * @param {string} name 11 | * @returns {DBTable} 12 | */ 13 | // eslint-disable-next-line no-unused-vars 14 | table(name) { 15 | throw new Error('Not implemented'); 16 | } 17 | 18 | /** 19 | * @param {string} path 20 | * @param {{name: string, type: string}[]} schema 21 | * @param {string} [name] 22 | * @returns {DBTable} 23 | */ 24 | // eslint-disable-next-line no-unused-vars 25 | fromCSV(path, schema, name) { 26 | throw new Error('Not implemented'); 27 | } 28 | 29 | /** 30 | * @param {import('arquero').internal.Table} table 31 | * @param {string} [name] 32 | * @returns {DBTable} 33 | */ 34 | // eslint-disable-next-line no-unused-vars 35 | fromArquero(table, name) { 36 | throw new Error('Not implemented'); 37 | } 38 | 39 | async close() { 40 | throw new Error('Not implemented'); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/databases/index.js: -------------------------------------------------------------------------------- 1 | export {Database} from './database'; 2 | export {TableView} from './table-view'; 3 | -------------------------------------------------------------------------------- /src/databases/postgres/index.js: -------------------------------------------------------------------------------- 1 | export {PostgresDatabase} from './pg-database'; 2 | export {PostgresTableView} from './pg-table-view'; 3 | -------------------------------------------------------------------------------- /src/databases/postgres/pg-code-gen.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('./pg-table-view').PosgresTableView} PosgresTableView */ 2 | 3 | import isString from 'arquero/src/util/is-string'; 4 | import createColumn from './utils/create-column'; 5 | import {GB_KEY} from './verbs/groupby'; 6 | import {genExpr} from './visitors/gen-expr'; 7 | 8 | /** 9 | * 10 | * @param {PosgresTableView|string} query 11 | * @param {string} [indentStr] 12 | * @param {number} [indentLvl] 13 | * @param {Counter} [counter] 14 | */ 15 | export default function postgresCodeGen(query, indentStr = ' ', indentLvl = 0, counter = new Counter()) { 16 | /** @type {string[]} */ 17 | const code = []; 18 | const indent = _indent(indentLvl, indentStr).join(''); 19 | const nl = indentStr ? '\n' : ' '; 20 | 21 | if (isString(query)) { 22 | code.push(indent); 23 | code.push(query, nl); 24 | return code.join(''); 25 | } 26 | 27 | /** @type {PosgresTableView} */ 28 | const {_clauses, _columns, _group, _order} = query; 29 | 30 | const tables = ['table' + counter.next(), _clauses.join ? 'table' + counter.next() : null]; 31 | const partition = _group && _group.map(GB_KEY).join(','); 32 | const order = _order && genOrderClause(_order, {partition, tables}); 33 | 34 | const opt = {partition, order, tables}; 35 | 36 | // SELECT 37 | code.push(indent); 38 | code.push('SELECT '); 39 | const select = _clauses.select || [ 40 | ..._columns.map(c => createColumn(c)), 41 | ...(_group || []).map(c => createColumn(GB_KEY(c))), 42 | ]; 43 | const select_str = select 44 | .map(({as, ...s}) => { 45 | const expr = genExpr(s, {...opt, withoutOver: !!_clauses.groupby}); 46 | const _as = as ? ` AS ${as}` : ''; 47 | return expr + _as; 48 | }) 49 | .join(','); 50 | code.push(...select_str, nl); 51 | 52 | // FROM 53 | code.push(indent); 54 | code.push('FROM '); 55 | if (typeof query._source === 'string') { 56 | code.push(query._source); 57 | } else { 58 | code.push('(', nl); 59 | code.push(postgresCodeGen(query._source, indentStr, indentLvl + 1, counter)); 60 | code.push(indent); 61 | code.push(')'); 62 | } 63 | code.push(' AS ', tables[0]); 64 | if (_clauses.join) { 65 | code.push(' ', _clauses.join.join_type, ' JOIN '); 66 | if (typeof _clauses.join.other === 'string') { 67 | code.push(_clauses.join.other); 68 | } else { 69 | code.push('(', nl); 70 | code.push(postgresCodeGen(_clauses.join.other, indentStr, indentLvl + 1, counter)); 71 | code.push(indent); 72 | code.push(')'); 73 | } 74 | code.push(' AS ', tables[1]); 75 | if (_clauses.join.on) { 76 | code.push(' ON '); 77 | code.push(genExpr(_clauses.join.on, opt)); 78 | } 79 | } 80 | code.push(nl); 81 | 82 | // WHERE 83 | if (_clauses.where) { 84 | code.push(indent); 85 | code.push('WHERE '); 86 | code.push(genExprList(_clauses.where, opt, ' AND ')); 87 | code.push(nl); 88 | } 89 | 90 | // GROUP BY 91 | if (Array.isArray(_clauses.groupby)) { 92 | code.push(indent); 93 | code.push('GROUP BY '); 94 | code.push(genExprList(_clauses.groupby, opt, ',')); 95 | code.push(nl); 96 | } 97 | 98 | // HAVING 99 | if (_clauses.having) { 100 | code.push(indent); 101 | code.push('HAVING '); 102 | code.push(genExprList(_clauses.having, opt, ' AND ')); 103 | code.push(nl); 104 | } 105 | 106 | // ORDER BY 107 | if (_clauses.orderby) { 108 | code.push(indent); 109 | code.push('ORDER BY '); 110 | code.push(genOrderClause(_clauses.orderby, {partition, tables})); 111 | code.push(nl); 112 | } 113 | 114 | // LIMIT 115 | if (_clauses.limit || _clauses.limit === 0) { 116 | code.push(indent); 117 | code.push('LIMIT ', _clauses.limit, nl); 118 | } 119 | 120 | // SET VERBS 121 | ['concat', 'except', 'intersect', 'union'] 122 | .filter(verb => _clauses[verb]) 123 | .forEach(verb => { 124 | _clauses[verb].forEach(q => { 125 | code.push(indent); 126 | code.push(verb.toUpperCase(), ' (', nl); 127 | code.push(postgresCodeGen(q, indentStr, indentLvl + 1, counter)); 128 | code.push(')', nl); 129 | }); 130 | }); 131 | 132 | return code.join(''); 133 | } 134 | 135 | export class Counter { 136 | constructor() { 137 | this.counter = 0; 138 | } 139 | 140 | next() { 141 | return this.counter++; 142 | } 143 | } 144 | 145 | /** 146 | * 147 | * @param {import('./pg-table-view').OrderInfo} orderby 148 | * @param {import('../../visitors/gen-expr').GenExprOpt} opt 149 | * @returns {string} 150 | */ 151 | function genOrderClause(orderby, opt) { 152 | const {exprs, descs} = orderby; 153 | return exprs.map((g, i) => genExpr(g, opt) + (descs[i] ? ' DESC' : '')).join(','); 154 | } 155 | 156 | /** 157 | * 158 | * @param {import('./pg-table-view').AstNode[]} list 159 | * @param {import('../../visitors/gen-expr').GenExprOpt} opt 160 | * @param {string} delim 161 | */ 162 | function genExprList(list, opt, delim) { 163 | return list.map(n => genExpr(n, opt)).join(delim); 164 | } 165 | 166 | /** 167 | * 168 | * @param {number} indentLvl 169 | * @param {string} indentStr 170 | */ 171 | function _indent(indentLvl, indentStr) { 172 | const ret = []; 173 | for (let i = 0; i < indentLvl; i++) { 174 | ret.push(indentStr); 175 | } 176 | return ret; 177 | } 178 | -------------------------------------------------------------------------------- /src/databases/postgres/pg-database.js: -------------------------------------------------------------------------------- 1 | import {Pool} from 'pg'; 2 | import {v4 as uuid} from 'uuid'; 3 | import * as fs from 'fs'; 4 | import * as fastcsv from 'fast-csv'; 5 | import {DBTable} from '../../db-table'; 6 | import {Database} from '../database'; 7 | import {PostgresTableView} from './pg-table-view'; 8 | 9 | /** @typedef {'TEXT' | 'BOOLEAN' | 'JSONB' | 'TIMESTAMPZ' | 'DOUBLE PRECISION'} PGType */ 10 | 11 | /** 12 | * @param {any} value 13 | * @returns {PGType | null} 14 | */ 15 | function getPGType(value) { 16 | if (value === null || value === undefined) { 17 | return null; 18 | } else if (typeof value === 'string') { 19 | return 'TEXT'; 20 | } else if (typeof value === 'number') { 21 | return 'DOUBLE PRECISION'; 22 | } else if (typeof value === 'boolean') { 23 | return 'BOOLEAN'; 24 | } else if (value instanceof Date) { 25 | return 'TIMESTAMPZ'; 26 | } else { 27 | return 'JSONB'; 28 | } 29 | } 30 | 31 | /** 32 | * @param {string} name 33 | * @param {string[]} cols 34 | * @returns {string} 35 | */ 36 | function insertInto(name, cols) { 37 | const vals = cols.map((_, i) => '$' + (i + 1)); 38 | return `INSERT INTO ${name} (${cols.join(',')}) VALUES (${vals.join(',')})`; 39 | } 40 | 41 | export class PostgresDatabase extends Database { 42 | /** 43 | * @typedef {object} PostgresCredential 44 | * @prop {string} user username 45 | * @prop {string} host host name 46 | * @prop {string} database database name 47 | * @prop {string} password password 48 | * @prop {number} port port 49 | */ 50 | 51 | /** 52 | * @param {PostgresCredential} credential 53 | */ 54 | constructor({user, host, database, password, port}) { 55 | super(); 56 | 57 | /** @type {Pool} */ 58 | this._pool = new Pool({user, host, database, password, port}); 59 | } 60 | 61 | /** 62 | * @param {string} name 63 | * @returns {DBTable} 64 | */ 65 | table(name) { 66 | const pbuilder = this.getColumnNames(name).then( 67 | colNames => new PostgresTableView(name, colNames, null, null, null, this), 68 | ); 69 | return new DBTable(pbuilder); 70 | } 71 | 72 | /** 73 | * @param {string} path 74 | * @param {{name: string, type: PGType}[]} schema 75 | * @param {string} [name] 76 | * @returns {DBTable} 77 | */ 78 | fromCSV(path, schema, name) { 79 | name = name || `__aq__table__${uuid().split('-').join('')}__`; 80 | const columnNames = schema.map(({name}) => name); 81 | const results = Promise.resolve() 82 | .then(() => { 83 | const stream = fs.createReadStream(path); 84 | const csvData = []; 85 | const csvStream = fastcsv 86 | .parse() 87 | .on('data', csvData.push) 88 | .on('end', () => csvData.shift()); 89 | stream.pipe(csvStream); 90 | return csvData; 91 | }) 92 | .then(async csvData => { 93 | await this.query(`CREATE TABLE ${name} (${schema.map(({name, type}) => name + ' ' + type).join(',')})`); 94 | 95 | const query = insertInto(name, columnNames); 96 | await this.query('BEGIN'); 97 | for (const row in csvData) { 98 | await this.query(query, row); 99 | } 100 | return this.query('COMMIT'); 101 | }); 102 | 103 | return getTableAfter(this, results, name); 104 | } 105 | 106 | /** 107 | * @param {import('arquero').internal.Table} table 108 | * @param {string} [name] 109 | * @returns {DBTable} 110 | */ 111 | fromArquero(table, name) { 112 | name = name || `__aq__table__${uuid().split('-').join('')}__`; 113 | const columnNames = table.columnNames(); 114 | const numRows = table.numRows(); 115 | const results = Promise.resolve() 116 | .then(() => 117 | columnNames.map(cn => { 118 | const column = table.getter(cn); 119 | for (let j = 0; j < numRows; j++) { 120 | const val = column(j); 121 | const type = getPGType(val); 122 | if (type !== null) { 123 | return type; 124 | } 125 | } 126 | return 'TEXT'; 127 | }), 128 | ) 129 | .then(async types => { 130 | await this.query(`CREATE TABLE ${name} (${columnNames.map((cn, i) => cn + ' ' + types[i]).join(',')})`); 131 | 132 | const insert = insertInto(name, columnNames); 133 | await this.query('BEGIN'); 134 | for (const row of table) { 135 | /** @type {string[]} */ 136 | const values = []; 137 | for (const i in columnNames) { 138 | const cn = columnNames[i]; 139 | const value = row[cn]; 140 | values.push(value); 141 | 142 | const type = getPGType(value); 143 | if (types[i] !== type && type !== null) { 144 | throw new Error('types in column ' + cn + ' do not match'); 145 | } 146 | } 147 | await this.query(insert, values); 148 | } 149 | return this.query('COMMIT'); 150 | }); 151 | 152 | return getTableAfter(this, results, name); 153 | } 154 | 155 | /** 156 | * @param {string} table 157 | * @returns {Promise} 158 | */ 159 | async getColumnNames(table) { 160 | return this._pool 161 | .query('SELECT column_name FROM information_schema.columns WHERE table_name = $1', [table]) 162 | .then(result => result.rows.map(r => r.column_name)); 163 | } 164 | 165 | /** 166 | * @param {string} text 167 | * @param {string[]} [values] 168 | * @returns {Promise} 169 | */ 170 | async query(text, values) { 171 | values = values || []; 172 | return this._pool.query(text, values); 173 | } 174 | 175 | async close() { 176 | await this._pool.end(); 177 | } 178 | } 179 | 180 | /** 181 | * @param {Database} db 182 | * @param {Promise} promise 183 | * @param {string} name 184 | */ 185 | function getTableAfter(db, promise, name) { 186 | const pbuilder = promise 187 | .then(() => db.getColumnNames(name)) 188 | .then(colNames => new PostgresTableView(name, colNames, null, null, null, db)); 189 | return new DBTable(pbuilder); 190 | } 191 | -------------------------------------------------------------------------------- /src/databases/postgres/pg-optimizer.js: -------------------------------------------------------------------------------- 1 | import {SqlQuery} from './sql-query'; 2 | 3 | const CLAUSE_EXEC_ORDER = [ 4 | 'where', 5 | 'groupby', 6 | 'having', 7 | 'select', 8 | 'distinct', 9 | 'orderby', 10 | 'limit', 11 | 'concat', 12 | 'union', 13 | 'intersect', 14 | 'except', 15 | ]; 16 | 17 | /** 18 | * optimize query 19 | * @param {SqlQuery} query a query to be optimized 20 | * @returns {SqlQuery} optimized query 21 | */ 22 | export function optimize(query) { 23 | if (typeof query === 'string' || typeof query._source === 'string') { 24 | return query; 25 | } 26 | 27 | const source = optimize(query._source); 28 | const keys = Object.keys(query._clauses); 29 | if (['concat', 'intersect', 'union', 'except'].some(clause => keys.includes(clause))) { 30 | return new SqlQuery(source, query._columns, query._clauses, query._group); 31 | } 32 | 33 | const source_keys = Object.keys(source._clauses); 34 | // fuse filter 35 | const source_highest_key = Math.max(...source_keys.map(key => CLAUSE_EXEC_ORDER.indexOf(key)), 0); 36 | if (source_highest_key === 0) { 37 | const where = [...(query._clauses.where || []), ...(source._clauses.where || [])]; 38 | return new SqlQuery( 39 | source._source, 40 | query._columns, 41 | {...query._clauses, ...(where.length === 0 ? {} : {where})}, 42 | query._group, 43 | ); 44 | } 45 | 46 | // genearl fuse clauses 47 | const query_lowest_key = Math.min(...keys.map(key => CLAUSE_EXEC_ORDER.indexOf(key)), 10000); 48 | if (query_lowest_key > source_highest_key) { 49 | return new SqlQuery(source._source, query._columns, {...source._clauses, ...query._clauses}, query._group); 50 | } 51 | 52 | // fuse select 53 | if (query_lowest_key === 3 && source_highest_key === 3) { 54 | if (source._clauses.select.every(s => s.type === 'Column' && !('as' in s))) 55 | return new SqlQuery(source._source, query._columns, {...source._clauses, ...query._clauses}, query._group); 56 | } 57 | 58 | // do not fuse 59 | return new SqlQuery(source, query._columns, query._clauses, query._group); 60 | } 61 | -------------------------------------------------------------------------------- /src/databases/postgres/pg-table-view.js: -------------------------------------------------------------------------------- 1 | import {all} from 'arquero'; 2 | import {TableView} from '../table-view'; 3 | import isFunction from 'arquero/src/util/is-function'; 4 | import verbs from './verbs'; 5 | import postgresCodeGen from './pg-code-gen'; 6 | 7 | export class PostgresTableView extends TableView { 8 | /** 9 | * @param {Source} source source table or another sql query 10 | * @param {string[]} schema object of table schema 11 | * @param {Clauses} [clauses] object of sql clauses 12 | * @param {string[]} [group] 13 | * @param {OrderInfo} [order] 14 | * @param {import('./pg-database').PostgresDatabase} [database] 15 | */ 16 | constructor(source, schema, clauses, group, order, database) { 17 | super(); 18 | 19 | /** @type {Source} */ 20 | this._source = source; 21 | 22 | /** @type {string[]} */ 23 | this._columns = schema; 24 | 25 | if (typeof source !== 'string') { 26 | database = source._database; 27 | } 28 | /** @type {import('./pg-database').PostgresDatabase} */ 29 | this._database = database; 30 | 31 | /** @type {Clauses} */ 32 | this._clauses = clauses || {}; 33 | 34 | /** @type {string[]} */ 35 | this._group = group; 36 | 37 | /** @type {OrderInfo} */ 38 | this._order = order; 39 | } 40 | 41 | /** 42 | * 43 | * @typedef {object} WrapParams 44 | * @prop {string[] | (s: string[]) => string[]} columns 45 | * @prop {Clauses | (c: Clauses) => Clauses} clauses 46 | * @prop {string[] | (s: string[]) => string[]} group 47 | * @prop {OrderInfo[] | (o: OrderInfo[]) => OrderInfo[]} order 48 | */ 49 | 50 | /** 51 | * 52 | * @param {WrapParams} param0 53 | */ 54 | _append({columns, clauses, group, order}) { 55 | return new PostgresTableView( 56 | this._source, 57 | columns !== undefined ? (isFunction(columns) ? columns(this._columns) : columns) : this._columns, 58 | clauses !== undefined ? (isFunction(clauses) ? clauses(this._clauses) : clauses) : this._clauses, 59 | group != undefined ? (isFunction(group) ? group(this._group) : group) : this._group, 60 | order != undefined ? (isFunction(order) ? order(this._order) : order) : this._order, 61 | ); 62 | } 63 | 64 | /** 65 | * 66 | * @param {WrapParams} param0 67 | */ 68 | _wrap({columns, clauses, group, order}) { 69 | return new PostgresTableView( 70 | this, 71 | columns !== undefined ? (isFunction(columns) ? columns(this._columns) : columns) : this._columns, 72 | clauses !== undefined ? (isFunction(clauses) ? clauses(this._clauses) : clauses) : {}, 73 | group !== undefined ? (isFunction(group) ? group(this._group) : group) : this._group, 74 | order !== undefined ? (isFunction(order) ? order(this._order) : order) : this._order, 75 | ); 76 | } 77 | 78 | /** 79 | * Indicates if the table has a groupby specification. 80 | * @return {boolean} True if grouped, false otherwise. 81 | */ 82 | isGrouped() { 83 | return !!this._group; 84 | } 85 | 86 | /** 87 | * Filter function invoked for each column name. 88 | * @callback NameFilter 89 | * @param {string} name The column name. 90 | * @param {number} index The column index. 91 | * @param {string[]} array The array of names. 92 | * @return {boolean} Returns true to retain the column name. 93 | */ 94 | 95 | /** 96 | * The table column names, optionally filtered. 97 | * @param {NameFilter} [filter] An optional filter function. 98 | * If unspecified, all column names are returned. 99 | * @return {string[]} An array of matching column names. 100 | */ 101 | columnNames(filter) { 102 | return filter ? this._columns.filter(filter) : this._columns.slice(); 103 | } 104 | 105 | /** 106 | * The column name at the given index. 107 | * @param {number} index The column index. 108 | * @return {string} The column name, 109 | * or undefined if the index is out of range. 110 | */ 111 | columnName(index) { 112 | return this._columns[index]; 113 | } 114 | 115 | // eslint-disable-next-line no-unused-vars 116 | column(name) { 117 | return []; 118 | } 119 | 120 | _sql() { 121 | return postgresCodeGen( 122 | this.ungroup() 123 | .select(all()) 124 | ._append({clauses: c => ({...c, orderby: this._order}), order: null}), 125 | ); 126 | } 127 | 128 | /** 129 | * @param {import('arquero/src/table/table').ObjectsOptions} [options] 130 | */ 131 | async objects(options = {}) { 132 | const {grouped, limit, offset} = options; 133 | 134 | if (grouped) { 135 | throw new Error('TODO: support output grouped table'); 136 | } 137 | 138 | let t = this; 139 | if (limit !== undefined) { 140 | t = t._append({clauses: c => ({...c, limit: options.limit})}); 141 | } 142 | if (offset !== undefined) { 143 | t = t._append({clauses: c => ({...c, offset: offset})}); 144 | } 145 | 146 | const results = await t._database.query(t._sql()); 147 | return results.rows; 148 | } 149 | } 150 | 151 | Object.assign(PostgresTableView.prototype, verbs); 152 | 153 | /** @typedef {string | PostgresTableView} Source _source in PostgresTableView */ 154 | 155 | /** 156 | * @typedef {object} AstNode 157 | * @prop {string} type 158 | */ 159 | 160 | /** 161 | * @typedef {'INNER' | 'LEFT' | 'RIGHT' | "FULL"} JoinType 162 | */ 163 | 164 | /** 165 | * @typedef {object} JoinInfo 166 | * @prop {AstNode} on 167 | * @prop {PostgresTableView} other 168 | * @prop {JoinType} join_type 169 | */ 170 | 171 | /** 172 | * @typedef {object} OrderInfo 173 | * @prop {AstNode[]} exprs 174 | * @prop {boolean[]} descs 175 | */ 176 | 177 | /** 178 | * @typedef {object} Clauses _clauses in PostgresTableView 179 | * @prop {AstNode[]} [select] 180 | * @prop {AstNode[]} [where] 181 | * @prop {AstNode[] | boolean} [groupby] 182 | * @prop {AstNode[]} [having] 183 | * @prop {JoinInfo} [join] 184 | * @prop {OrderInfo} [orderby] 185 | * @prop {number} [limit] 186 | * @prop {number} [offset] 187 | * @prop {Source[]} [concat] 188 | * @prop {Source[]} [union] 189 | * @prop {Source[]} [intersect] 190 | * @prop {Source[]} [except] 191 | */ 192 | 193 | /** 194 | * @typedef {object} Schema _schema in PostgresTableView 195 | * @prop {string[]} columns 196 | * @prop {string[]} [groupby] 197 | */ 198 | -------------------------------------------------------------------------------- /src/databases/postgres/utils/compose-queries.js: -------------------------------------------------------------------------------- 1 | /** 2 | * compose a list of queries with `verb` 3 | * @param {'union' | 'intersect' | 'except' | 'concat'} verb a verb to compose the list of queries 4 | * @param {SqlQuery} queries the list of queries to be composed 5 | * @returns {string} composed queries 6 | */ 7 | export default function (verb, queries) { 8 | return queries.map(nameOrSqlQueryToSql).join(verb + '\n'); 9 | } 10 | 11 | /** 12 | * to SQL representation of the `table` 13 | * @param {Source} table table to be converted to SQL 14 | * @returns {string} SQL string of the table 15 | */ 16 | export function nameOrSqlQueryToSql(table) { 17 | if (typeof table === 'string') { 18 | return `SELECT *\nFROM ${table}\n`; 19 | } else { 20 | return table.toSql(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/databases/postgres/utils/create-column.js: -------------------------------------------------------------------------------- 1 | /** @typedef { {type: 'Column', name: string, as?: string, table?: number} } ColumnType */ 2 | 3 | /** 4 | * create a column ast node with the name `name` 5 | * @param {string} name input name of the column 6 | * @param {string} [as] output name of the column 7 | * @param {number} [table] table identifier 8 | * @returns {ColumnType} a column ast node with input name `name` and output namd `as` 9 | */ 10 | export default function (name, as, table) { 11 | const _as = as && as !== name ? {as} : {}; 12 | const _table = table ? {table} : {}; 13 | return {type: 'Column', name, ..._as, ..._table}; 14 | } 15 | -------------------------------------------------------------------------------- /src/databases/postgres/utils/is-reserved.js: -------------------------------------------------------------------------------- 1 | export const ARQUERO_SQL_PREFIX = '___arquero_sql_'; 2 | export const ARQUERO_SQL_SUFFIX = '___'; 3 | 4 | /** 5 | * 6 | * @param {string} name 7 | */ 8 | export default function (name) { 9 | return name.startsWith(ARQUERO_SQL_PREFIX) && name.endsWith(ARQUERO_SQL_SUFFIX); 10 | } 11 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/common.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('arquero').internal.Table} Table */ 2 | /** @typedef {import('../pg-table-view').PostgresTableView} PostgresTableView */ 3 | 4 | import createColumn from '../utils/create-column'; 5 | 6 | /** 7 | * 8 | * @param {'concat' | 'except' | 'intersect' | 'union'} verb 9 | * @returns {(query: PostgresTableView, others: PostgresTableView[]) => PostgresTableView} 10 | */ 11 | export function set_verb(verb) { 12 | return (table, others) => { 13 | const select = table.columnNames().map(col => createColumn(col)); 14 | const tables = others.map(other => other.ungroup()); 15 | return table.ungroup()._wrap({clauses: {select, [verb]: tables}}); 16 | }; 17 | } 18 | 19 | /** 20 | * @typedef {object} Verb Arquero's verb object 21 | * @prop {string} verb 22 | * @prop {object} schema 23 | * @prop {(table: Table, catalog: Function) => Table} evaluate 24 | * @prop {() => object} toObject 25 | * @prop {() => object} toAST 26 | */ 27 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/concat.js: -------------------------------------------------------------------------------- 1 | import {set_verb} from './common'; 2 | export default set_verb('concat'); 3 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/dedupe.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('../pg-table-view').PostgresTableView} PostgresTableView */ 2 | /** @typedef {import('arquero/src/table/transformable').ListEntry} ListEntry */ 3 | 4 | import {not, op} from 'arquero'; 5 | 6 | const ROW_NUMBER = '___arquero_sql_row_number___'; 7 | 8 | /** 9 | * 10 | * @param {PostgresTableView} table 11 | * @param {ListEntry[]} keys 12 | * @returns {PostgresTableView} 13 | */ 14 | export default function (table, keys = []) { 15 | return table 16 | .groupby(...(keys.length ? keys : table.columnNames())) 17 | .filter(() => op.row_number() === 1) 18 | .ungroup() 19 | .select(not(ROW_NUMBER)); 20 | } 21 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/derive.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('./common').Verb} Verb */ 2 | /** @typedef {import('../pg-table-view').PostgresTableView} PostgresTableView */ 3 | 4 | import {internal} from 'arquero'; 5 | import error from 'arquero/src/util/error'; 6 | import createColumn from '../utils/create-column'; 7 | import {ARQUERO_AGGREGATION_FN, ARQUERO_WINDOW_FN} from '../visitors/gen-expr'; 8 | import hasFunction from '../visitors/has-function'; 9 | import {GB_KEY} from './groupby'; 10 | 11 | /** 12 | * 13 | * @param {PostgresTableView} table 14 | * @param {import('arquero/src/table/transformable').ExprObject} values 15 | * @param {import('arquero/src/table/transformable').DeriveOptions} [options] 16 | * @returns {PostgresTableView} 17 | */ 18 | export default function (table, values, options = {}) { 19 | if (Object.keys(options).length > 0) { 20 | error("Arquero-SQL does not support derive's option"); 21 | } 22 | 23 | /** @type {Map} */ 24 | const columns = new Map(); 25 | const {exprs, names} = internal.parse(values, {ast: true}); 26 | table.columnNames().forEach(columnName => columns.set(columnName, createColumn(columnName))); 27 | exprs.forEach((expr, idx) => { 28 | if ([ARQUERO_AGGREGATION_FN, ARQUERO_WINDOW_FN].every(f => hasFunction(expr, f))) { 29 | error('Cannot derive an expression containing both an aggregation function and a window fundtion'); 30 | } 31 | 32 | /** @type {string} */ 33 | const as = names[idx]; 34 | columns.set(as, {...expr, as}); 35 | }); 36 | 37 | if (table.isGrouped()) { 38 | console.warn('Deriving with group may produce output with different ordering of rows'); 39 | 40 | if ([...columns.values()].some(v => hasFunction(v, ARQUERO_WINDOW_FN))) { 41 | console.warn( 42 | 'Deriving with window functions with group and without and explicit ordering may produce different result than Arquero', 43 | ); 44 | } 45 | } 46 | 47 | let groupby_cols = []; 48 | if (table.isGrouped()) { 49 | groupby_cols = table._group.map(key => createColumn(GB_KEY(key))); 50 | } 51 | return table._wrap({ 52 | clauses: {select: [...columns.values(), ...groupby_cols]}, 53 | columns: [...columns.keys()], 54 | }); 55 | } 56 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/except.js: -------------------------------------------------------------------------------- 1 | import {set_verb} from './common'; 2 | export default set_verb('except'); 3 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/filter.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('./common').Verb} Verb */ 2 | /** @typedef {import('../pg-table-view').PostgresTableView} PostgresTableView */ 3 | 4 | import {internal, not} from 'arquero'; 5 | import {ARQUERO_AGGREGATION_FN, ARQUERO_WINDOW_FN} from '../visitors/gen-expr'; 6 | import hasFunction from '../visitors/has-function'; 7 | 8 | const TMP_COL = '___arquero_sql_predicate___'; 9 | 10 | /** 11 | * 12 | * @param {PostgresTableView} table 13 | * @param {import('arquero/dist/types/table/transformable').TableExpr|string} criteria 14 | * @returns {PostgresTableView} 15 | */ 16 | export default function (table, criteria) { 17 | const _criteria = internal.parse({p: criteria}, {ast: true}).exprs[0]; 18 | 19 | if (!hasFunction(_criteria, [...ARQUERO_AGGREGATION_FN, ...ARQUERO_WINDOW_FN])) { 20 | return table._wrap({clauses: {where: [_criteria]}}); 21 | } 22 | 23 | return table 24 | .derive({[TMP_COL]: criteria}) 25 | .filter(`d => d.${TMP_COL}`) 26 | .select(not(TMP_COL)); 27 | } 28 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/fold.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('../pg-table-view').PostgresTableView} PostgresTableView */ 2 | /** @typedef {import('arquero/src/table/transformable').ExprList} ExprList */ 3 | /** @typedef {import('arquero/src/table/transformable').FoldOptions} FoldOptions */ 4 | 5 | import parse from 'arquero/src/verbs/util/parse'; 6 | import createColumn from '../utils/create-column'; 7 | 8 | 9 | /** 10 | * @param {PostgresTableView} table 11 | * @param {ExprList} values 12 | * @param {FoldOptions} [options] 13 | */ 14 | export default function(table, values, options = {}) { 15 | const [k = 'key', v = 'value'] = options.as || []; 16 | const {names, exprs, ops} = parse('fold', table, values, {ast: true}); 17 | if (ops && ops.length) { 18 | throw new Error('TODO: support ops from parse'); 19 | } 20 | 21 | table = table.ungroup(); 22 | const otherColumns = table._columns.filter(c => !names.includes(c)); 23 | 24 | let _expr = {type: 'Literal', raw: 'null'}; 25 | for (const i in names) { 26 | const name = names[i]; 27 | const expr = exprs[i]; 28 | 29 | _expr = { 30 | type: 'ConditionalExpression', 31 | test: { 32 | type: 'LogicalExpression', 33 | left: {type: 'Constant', raw: "'" + name + "'"}, 34 | right: {type: 'Constant', raw: '__aq__fold__key__'}, 35 | operator: '===', 36 | }, 37 | consequent: expr, 38 | alternate: _expr, 39 | }; 40 | } 41 | 42 | return table._wrap({ 43 | columns: [...otherColumns, k, v], 44 | clauses: { 45 | select: [ 46 | ...otherColumns.map(c => createColumn(c)), 47 | createColumn('__aq__fold__key__', k), 48 | {..._expr, as: v}, 49 | ], 50 | join: { 51 | other: `(SELECT unnest(ARRAY[${names.map(name => "'" + name + "'").join(',')}]) __aq__fold__key__)`, 52 | }, 53 | } 54 | }); 55 | } -------------------------------------------------------------------------------- /src/databases/postgres/verbs/groupby.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('./common').Verb} Verb */ 2 | /** @typedef {import('../pg-table-view').PostgresTableView} PostgresTableView */ 3 | 4 | import resolve from 'arquero/src/helpers/selection'; 5 | import isFunction from 'arquero/src/util/is-function'; 6 | import isNumber from 'arquero/src/util/is-number'; 7 | 8 | export const GB_KEY_PREFIX = '___arquero_sql_group_'; 9 | export const GB_KEY_SUFFIX = '___'; 10 | export const GB_KEY = key => GB_KEY_PREFIX + key + GB_KEY_SUFFIX; 11 | 12 | /** 13 | * 14 | * @param {PostgresTableView} table 15 | * @param {import('arquero/src/table/transformable').ListEntry[]} values 16 | * @returns {PostgresTableView} 17 | */ 18 | export default function (table, values) { 19 | if (table.isGrouped()) { 20 | table = table.ungroup(); 21 | } 22 | 23 | // TODO: use Arquero's parse function? 24 | 25 | const _keys = {}; 26 | values.forEach(key => { 27 | if (isFunction(key)) { 28 | // selection 29 | const sel = resolve(table, key); 30 | sel.forEach((v, k) => (_keys[GB_KEY(k)] = `d => d["${v}"]`)); 31 | } else if (typeof key === 'object') { 32 | // derive 33 | Object.entries(key).forEach(([k, v]) => (_keys[GB_KEY(k)] = v)); 34 | } else { 35 | // column 36 | key = isNumber(key) ? table.columnName(key) : key; 37 | _keys[GB_KEY(key)] = `d => d["${key}"]`; 38 | } 39 | }); 40 | 41 | const group = Object.keys(_keys).map(key => key.substring(GB_KEY_PREFIX.length, key.length - GB_KEY_SUFFIX.length)); 42 | return table.derive(_keys)._append({group, columns: table.columnNames()}); 43 | } 44 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/index.js: -------------------------------------------------------------------------------- 1 | import __concat from './concat'; 2 | import __dedupe from './dedupe'; 3 | import __derive from './derive'; 4 | import __except from './except'; 5 | import __filter from './filter'; 6 | import __fold from './fold'; 7 | import __groupby from './groupby'; 8 | import __intersect from './intersect'; 9 | import __join from './join'; 10 | import __orderby from './orderby'; 11 | import __rollup from './rollup'; 12 | import __sample from './sample'; 13 | import __select from './select'; 14 | import __ungroup from './ungroup'; 15 | import __union from './union'; 16 | import __unorder from './unorder'; 17 | 18 | import {op} from 'arquero'; 19 | 20 | export default { 21 | // __antijoin: (table, other, on) => 22 | // __semijoin(table, other, on, { anti: true }), 23 | __count: (table, options = {}) => __rollup(table, {[options.as || 'count']: op.count()}), 24 | __cross: (table, other, values, options) => 25 | __join(table, other, () => true, values, { 26 | ...options, 27 | left: true, 28 | right: true, 29 | }), 30 | __concat, 31 | __dedupe, 32 | __derive, 33 | __except, 34 | __filter, 35 | __fold, 36 | // __impute, 37 | __intersect, 38 | __join, 39 | // __lookup, 40 | // __pivot, 41 | // __relocate, 42 | // __rename, 43 | __rollup, 44 | __sample, 45 | __select, 46 | // __semijoin, 47 | // __spread, 48 | __union, 49 | // __unroll, 50 | __groupby, 51 | __orderby, 52 | __ungroup, 53 | __unorder, 54 | // __reduce, 55 | }; 56 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/intersect.js: -------------------------------------------------------------------------------- 1 | import {set_verb} from './common'; 2 | export default set_verb('intersect'); 3 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/join.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('./common').Verb} Verb */ 2 | /** @typedef {import('../pg-table-view').PostgresTableView} PostgresTableView */ 3 | 4 | import parseValue from 'arquero/src/verbs/util/parse'; 5 | import {inferKeys} from 'arquero/src/verbs/join'; 6 | import {all, not} from 'arquero'; 7 | import {internal} from 'arquero'; 8 | import isArray from 'arquero/src/util/is-array'; 9 | import isNumber from 'arquero/src/util/is-number'; 10 | import isString from 'arquero/src/util/is-string'; 11 | import toArray from 'arquero/src/util/to-array'; 12 | import toString from 'arquero/src/util/to-string'; 13 | 14 | /** @type {['INNER', 'RIGHT', 'LEFT', 'FULL']} */ 15 | export const JOIN_TYPES = ['INNER', 'RIGHT', 'LEFT', 'FULL']; 16 | 17 | const OPT_L = {aggregate: false, window: false}; 18 | const OPT_R = {...OPT_L, index: 1}; 19 | 20 | const optParse = {join: true, ast: true}; 21 | 22 | /** 23 | * 24 | * @param {PostgresTableView} tableL 25 | * @param {PostgresTableView} tableR 26 | * @param {import('arquero/src/table/transformable').JoinPredicate} on 27 | * @param {import('arquero/src/table/transformable').JoinValues} values 28 | * @param {import('arquero/src/table/transformable').JoinOptions} options 29 | * @returns {PostgresTableView} 30 | */ 31 | export default function (tableL, tableR, on, values, options = {}) { 32 | on = inferKeys(tableL, tableR, on); 33 | 34 | if (isArray(on)) { 35 | const [onL, onR] = on.map(toArray); 36 | if (onL.length !== onR.length) { 37 | throw new Error('Mismatched number of join keys'); 38 | } 39 | 40 | const body = onL 41 | .map((l, i) => { 42 | l = isNumber(l) ? tableL.columnName(l) : l; 43 | const r = isNumber(onR[i]) ? tableR.columnName(onR[i]) : l; 44 | return `a["${l}"] === b["${r}"]`; 45 | }) 46 | .join(' && '); 47 | on = `(a, b) => ${body}`; 48 | 49 | if (!values) { 50 | values = inferValues(tableL, onL, onR, options); 51 | } 52 | } else if (!values) { 53 | values = [all(), all()]; 54 | } 55 | on = internal.parse({on}, {ast: true, join: true}).exprs[0]; 56 | 57 | const {exprs, names} = parseValues(tableL, tableR, values, optParse, options.suffix); 58 | exprs.forEach((expr, i) => (expr.name === names[i] ? null : (expr.as = names[i]))); 59 | 60 | const join_type = JOIN_TYPES[(~~options.left << 1) + ~~options.right]; 61 | return tableL._wrap({ 62 | clauses: { 63 | select: exprs, 64 | join: {other: tableR, on, join_type}, 65 | }, 66 | columns: exprs.map(c => c.as || c.name), 67 | }); 68 | } 69 | 70 | /** 71 | * 72 | * @param {PostgresTableView} queryL 73 | * @param {import('arquero/src/table/transformable').JoinKey[]} onL 74 | * @param {import('arquero/src/table/transformable').JoinKey[]} onR 75 | * @param {import('arquero/src/table/transformable').JoinOptions} options 76 | * @returns {import('arquero/src/table/transformable').JoinKey} 77 | */ 78 | function inferValues(tableL, onL, onR, options) { 79 | const isect = []; 80 | onL.forEach((s, i) => (isString(s) && s === onR[i] ? isect.push(s) : 0)); 81 | const vR = not(isect); 82 | 83 | if (options.left && options.right) { 84 | // for full join, merge shared key columns together 85 | const shared = new Set(isect); 86 | return [ 87 | tableL.columnNames().map(s => { 88 | const c = `[${toString(s)}]`; 89 | return shared.has(s) ? {[s]: `(a, b) => op.equal(a${c}, null) ? b${c} : a${c}`} : s; 90 | }), 91 | vR, 92 | ]; 93 | } 94 | 95 | return options.right ? [vR, all()] : [all(), vR]; 96 | } 97 | 98 | /** 99 | * 100 | * @param {PostgresTableView} tableL 101 | * @param {PostgresTableView} tableR 102 | * @param {import('arquero/src/table/transformable').JoinValues} values 103 | * @param {object} optParse 104 | * @param {string[]} suffix 105 | */ 106 | function parseValues(tableL, tableR, values, optParse, suffix = []) { 107 | if (isArray(values)) { 108 | let vL, 109 | vR, 110 | vJ, 111 | n = values.length; 112 | vL = vR = vJ = {names: [], exprs: []}; 113 | 114 | if (n--) { 115 | vL = parseValue('join', tableL, values[0], optParse); 116 | // add table index 117 | // assignTable(vL.exprs, 1); 118 | } 119 | if (n--) { 120 | vR = parseValue('join', tableR, values[1], {...OPT_R, ...optParse}); 121 | // add table index 122 | assignTable(vR.exprs, 2); 123 | } 124 | if (n--) { 125 | vJ = internal.parse(values[2], optParse); 126 | } 127 | 128 | // handle name collisions 129 | const rename = new Set(); 130 | const namesL = new Set(vL.names); 131 | vR.names.forEach(name => { 132 | if (namesL.has(name)) { 133 | rename.add(name); 134 | } 135 | }); 136 | if (rename.size) { 137 | rekey(vL.names, rename, suffix[0] || '_1'); 138 | rekey(vR.names, rename, suffix[1] || '_2'); 139 | } 140 | 141 | return { 142 | names: vL.names.concat(vR.names, vJ.names), 143 | exprs: vL.exprs.concat(vR.exprs, vJ.exprs), 144 | }; 145 | } else { 146 | const v = internal.parse(values, optParse); 147 | assignTable(v, 1); 148 | return v; 149 | } 150 | } 151 | 152 | function rekey(names, rename, suffix) { 153 | names.forEach((name, i) => (rename.has(name) ? (names[i] = name + suffix) : 0)); 154 | } 155 | 156 | function assignTable(expr, index) { 157 | if (typeof expr !== 'object') return; 158 | 159 | if (expr.type === 'Column') { 160 | expr.table = index; 161 | } else { 162 | Object.values(expr).forEach(e => assignTable(e, index)); 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/orderby.js: -------------------------------------------------------------------------------- 1 | /** @typedef { import('../pg-table-view').PostgresTableView } PostgresTableView */ 2 | /** @typedef { import('./common').Verb} Verb */ 3 | 4 | import {internal} from 'arquero'; 5 | import error from 'arquero/src/util/error'; 6 | import isFunction from 'arquero/src/util/is-function'; 7 | import isNumber from 'arquero/src/util/is-number'; 8 | import isObject from 'arquero/src/util/is-object'; 9 | import isString from 'arquero/src/util/is-string'; 10 | 11 | /** 12 | * 13 | * @param {PostgresTableView} table 14 | * @param {import('arquero/src/table/transformable').OrderKey[]} keys 15 | * @returns {PostgresTableView} 16 | */ 17 | export default function (table, keys) { 18 | return table._wrap({order: parseValues(table, keys)}); 19 | } 20 | 21 | function parseValues(table, params) { 22 | let index = -1; 23 | const exprs = new Map(); 24 | const descs = []; 25 | const add = (val, desc) => (exprs.set(++index + '', val), descs.push(!!desc)); 26 | 27 | params.forEach(param => { 28 | const expr = param.expr != null ? param.expr : param; 29 | 30 | if (isObject(expr) && !isFunction(expr)) { 31 | for (const key in expr) { 32 | add(expr[key], param.desc); 33 | } 34 | } else { 35 | add( 36 | isNumber(expr) 37 | ? `d => d["${table.columnName(expr)}"]` 38 | : isString(expr) 39 | ? `d => d["${expr}"]` 40 | : isFunction(expr) 41 | ? param 42 | : error(`Invalid orderby field: ${param + ''}`), 43 | param.desc, 44 | ); 45 | } 46 | }); 47 | 48 | return {...internal.parse(exprs, {ast: true}), descs}; 49 | } 50 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/pivot.js: -------------------------------------------------------------------------------- 1 | /* 2 | Example: 3 | columns=[A, B, C, D] 4 | 5 | 1. PIVOT(A, B) 6 | 7 | find all possible keys: X, Y 8 | 9 | SELECT first(B) FILTER (WHERE A = 'X') as X, first(B) FILTER (WHERE A = 'Y') as Y 10 | FROM Table 11 | 12 | 13 | 2. GROUPBY(C) -> PIVOT(A, SUM(B)) 14 | 15 | find all possible keys: X, Y 16 | 17 | SELECT C, SUM(B) FILTER (WHERE A = 'X') as X, SUM(B) FILTER (WHERE A = 'Y') as Y 18 | FROM Table 19 | GROUPBY C 20 | */ 21 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/rollup.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('./common').Verb} Verb */ 2 | /** @typedef {import('../pg-table-view').PostgresTableView} PostgresTableView */ 3 | 4 | import {internal} from 'arquero'; 5 | import error from 'arquero/src/util/error'; 6 | import createColumn from '../utils/create-column'; 7 | import {ARQUERO_WINDOW_FN} from '../visitors/gen-expr'; 8 | import hasFunction from '../visitors/has-function'; 9 | import {GB_KEY} from './groupby'; 10 | 11 | /** 12 | * 13 | * @param {PostgresTableView} table 14 | * @param {import('arquero/src/table/transformable').ExprObject} [values] 15 | * @returns {PostgresTableView} 16 | */ 17 | export default function (table, values = []) { 18 | /** @type {Map} */ 19 | const columns = new Map(); 20 | if (table.isGrouped()) { 21 | table._group.forEach(key => columns.set(key, createColumn(GB_KEY(key), key))); 22 | } 23 | 24 | const {exprs, names} = internal.parse(values, {ast: true, argonly: true}); 25 | exprs.forEach((expr, idx) => { 26 | if (hasFunction(expr, ARQUERO_WINDOW_FN)) { 27 | error('Cannot rollup an expression containing a window fundtion'); 28 | } 29 | 30 | const as = names[idx]; 31 | columns.set(as, {...expr, as}); 32 | }); 33 | 34 | return table._wrap({ 35 | clauses: { 36 | select: [...columns.values()], 37 | groupby: !table.isGrouped() || table._group.map(g => createColumn(GB_KEY(g))), 38 | }, 39 | columns: [...columns.keys()], 40 | group: null, 41 | }); 42 | } 43 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/sample.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('./common').Verb} Verb */ 2 | /** @typedef {import('../pg-table-view').PostgresTableView} PostgresTableView */ 3 | 4 | import {op} from 'arquero'; 5 | 6 | /** 7 | * 8 | * @param {PostgresTableView} table 9 | * @param {number|import('arquero/src/table/transformable').TableExpr} size 10 | * @param {import('arquero/src/table/transformable').SampleOptions} options 11 | * @returns {PostgresTableView} 12 | */ 13 | export default function (table, size, options = {}) { 14 | if (typeof size !== 'number') { 15 | // TODO: calculate the size -> then use the calculated size as limit 16 | throw new Error('sample only support constant sample size'); 17 | } 18 | 19 | if (options.replace) { 20 | // TODO: create new table, randomly insert new comlumns into that table 21 | throw new Error("Arquero-SQL's sample does not support replace"); 22 | } 23 | 24 | if (options.weight) { 25 | throw new Error("Arquero-SQL's sample does not support weight"); 26 | } 27 | 28 | console.warn('Sampling will produce output with different ordering of rows'); 29 | return table 30 | .orderby([() => op.random()]) 31 | ._append({clauses: c => ({...c, limit: size})}) 32 | .unorder(); 33 | } 34 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/select.js: -------------------------------------------------------------------------------- 1 | /** @typedef { import('../pg-table-view').PostgresTableView } PostgresTableView */ 2 | /** @typedef { import('../utils/create-column').ColumnType } ColumnType */ 3 | /** @typedef { import('./common').Verb } Verb */ 4 | 5 | import createColumn from '../utils/create-column'; 6 | import resolve from 'arquero/src/helpers/selection'; 7 | import isString from 'arquero/src/util/is-string'; 8 | import {GB_KEY} from './groupby'; 9 | 10 | /** 11 | * 12 | * @param {PostgresTableView} table 13 | * @param {import('arquero/src/table/transformable').SelectEntry[]} columns 14 | * @returns {PostgresTableView} 15 | */ 16 | export default function (table, columns) { 17 | /** @type {ColumnType[]} */ 18 | const cols = []; 19 | resolve(table, columns).forEach((next, curr) => { 20 | next = isString(next) ? next : curr; 21 | if (next) { 22 | if (!table._columns.includes(curr)) { 23 | throw new Error(`Unrecognized column: ${curr}`); 24 | } 25 | cols.push(createColumn(curr, next)); 26 | } 27 | }); 28 | 29 | let groupby_cols = []; 30 | if (table.isGrouped()) { 31 | groupby_cols = table._group.map(key => createColumn(GB_KEY(key))); 32 | } 33 | 34 | return table._wrap({ 35 | clauses: {select: [...cols, ...groupby_cols]}, 36 | columns: cols.map(col => col.as || col.name), 37 | }); 38 | } 39 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/ungroup.js: -------------------------------------------------------------------------------- 1 | /** @typedef { import('../pg-table-view').PostgresTableView} PostgresTableView */ 2 | 3 | /** 4 | * 5 | * @param {PostgresTableView} table 6 | */ 7 | export default function (table) { 8 | return table.isGrouped() ? table._wrap({columns: table.columnNames(), group: null}) : table; 9 | } 10 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/union.js: -------------------------------------------------------------------------------- 1 | import {set_verb} from './common'; 2 | export default set_verb('union'); 3 | -------------------------------------------------------------------------------- /src/databases/postgres/verbs/unorder.js: -------------------------------------------------------------------------------- 1 | /** @typedef { import('../pg-table-view').PostgresTableView} PostgresTableView */ 2 | 3 | /** 4 | * 5 | * @param {PostgresTableView} table 6 | */ 7 | export default function (table) { 8 | return table._order ? table._wrap({order: null}) : table; 9 | } 10 | -------------------------------------------------------------------------------- /src/databases/postgres/visitors/aggregated-columns.js: -------------------------------------------------------------------------------- 1 | import {columns} from './columns'; 2 | 3 | export const aggregatedColumns = node => { 4 | return visitors[node.type](node); 5 | }; 6 | 7 | const binary = node => { 8 | return [...aggregatedColumns(node.left), ...aggregatedColumns(node.right)]; 9 | }; 10 | 11 | const call = node => { 12 | return [...aggregatedColumns(node.callee), ...list(node.arguments)]; 13 | }; 14 | 15 | const list = array => { 16 | return array.map(node => aggregatedColumns(node)).flat(); 17 | }; 18 | 19 | const AGGREGATED_OPS = ['mean', 'row_number']; 20 | 21 | const visitors = { 22 | Column: () => [], 23 | Constant: () => [], 24 | Function: node => (AGGREGATED_OPS.includes(node.name) ? node.arguments.map(a => columns(a)).flat() : []), 25 | Parameter: node => { 26 | throw new Error('Parameter is not supported: ' + JSON.stringify(node)); 27 | }, 28 | OpLookup: node => { 29 | throw new Error('OpLookup is not supported: ' + JSON.stringify(node)); 30 | }, 31 | Literal: () => false, 32 | Identifier: () => false, 33 | TemplateLiteral: node => node.expressions.map(e => aggregatedColumns(e)).flat(), 34 | MemberExpression: node => { 35 | throw new Error('MemberExpression is not supported: ' + JSON.stringify(node)); 36 | }, 37 | CallExpression: call, 38 | NewExpression: node => { 39 | throw new Error('NewExpression is not supported: ' + JSON.stringify(node)); 40 | }, 41 | ArrayExpression: node => { 42 | throw new Error('ArrayExpression is not supported: ' + JSON.stringify(node)); 43 | }, 44 | AssignmentExpression: node => { 45 | throw new Error('AssignmentExpression is not supported: ' + JSON.stringify(node)); 46 | }, 47 | BinaryExpression: binary, 48 | LogicalExpression: binary, 49 | UnaryExpression: node => aggregatedColumns(node.argument), 50 | ConditionalExpression: node => [ 51 | ...aggregatedColumns(node.test), 52 | ...aggregatedColumns(node.consequent), 53 | ...aggregatedColumns(node.alternate), 54 | ], 55 | ObjectExpression: node => { 56 | throw new Error('ObjectExpression is not supported: ' + JSON.stringify(node)); 57 | }, 58 | Property: node => { 59 | throw new Error('Property is not supported: ' + JSON.stringify(node)); 60 | }, 61 | 62 | ArrowFunctionExpression: node => { 63 | throw new Error('ArrowFunctionExpression is not supported: ' + JSON.stringify(node)); 64 | }, 65 | FunctionExpression: node => { 66 | throw new Error('FunctionExpression is not supported: ' + JSON.stringify(node)); 67 | }, 68 | FunctionDeclaration: node => { 69 | throw new Error('FunctionDeclaration is not supported: ' + JSON.stringify(node)); 70 | }, 71 | 72 | ArrayPattern: node => { 73 | throw new Error('ArrayPattern is not supported: ' + JSON.stringify(node)); 74 | }, 75 | ObjectPattern: node => { 76 | throw new Error('ObjectPattern is not supported: ' + JSON.stringify(node)); 77 | }, 78 | VariableDeclaration: node => { 79 | throw new Error('VariableDeclaration is not supported: ' + JSON.stringify(node)); 80 | }, 81 | VariableDeclarator: node => { 82 | throw new Error('VariableDeclarator is not supported: ' + JSON.stringify(node)); 83 | }, 84 | SpreadElement: node => { 85 | throw new Error('SpreadElement is not supported: ' + JSON.stringify(node)); 86 | }, 87 | 88 | BlockStatement: node => { 89 | throw new Error('BlockStatement is not supported: ' + JSON.stringify(node)); 90 | }, 91 | BreakStatement: node => { 92 | throw new Error('BreakStatement is not supported: ' + JSON.stringify(node)); 93 | }, 94 | ExpressionStatement: node => aggregatedColumns(node.expression), 95 | IfStatement: node => { 96 | throw new Error('IfStatement is not supported: ' + JSON.stringify(node)); 97 | }, 98 | SwitchStatement: node => { 99 | throw new Error('SwitchStatement is not supported: ' + JSON.stringify(node)); 100 | }, 101 | SwitchCase: node => { 102 | throw new Error('SwitchCase is not supported: ' + JSON.stringify(node)); 103 | }, 104 | ReturnStatement: node => { 105 | throw new Error('ReturnStatement is not supported: ' + JSON.stringify(node)); 106 | }, 107 | Program: node => { 108 | throw new Error('Program is not supported: ' + JSON.stringify(node)); 109 | }, 110 | }; 111 | -------------------------------------------------------------------------------- /src/databases/postgres/visitors/columns.js: -------------------------------------------------------------------------------- 1 | export const columns = node => { 2 | return visitors[node.type](node); 3 | }; 4 | 5 | const binary = node => { 6 | return [...columns(node.left), ...columns(node.right)]; 7 | }; 8 | 9 | const call = node => { 10 | return [...columns(node.callee), ...list(node.arguments)]; 11 | }; 12 | 13 | const list = array => { 14 | return array.map(node => columns(node)).flat(); 15 | }; 16 | 17 | const visitors = { 18 | Column: node => [node], 19 | Constant: () => [], 20 | Function: node => node.arguments.map(a => columns(a)).flat(), 21 | Parameter: node => { 22 | throw new Error('Parameter is not supported: ' + JSON.stringify(node)); 23 | }, 24 | OpLookup: node => { 25 | throw new Error('OpLookup is not supported: ' + JSON.stringify(node)); 26 | }, 27 | Literal: () => false, 28 | Identifier: () => false, 29 | TemplateLiteral: node => node.expressions.map(e => columns(e)).flat(), 30 | MemberExpression: node => { 31 | throw new Error('MemberExpression is not supported: ' + JSON.stringify(node)); 32 | }, 33 | CallExpression: call, 34 | NewExpression: node => { 35 | throw new Error('NewExpression is not supported: ' + JSON.stringify(node)); 36 | }, 37 | ArrayExpression: node => { 38 | throw new Error('ArrayExpression is not supported: ' + JSON.stringify(node)); 39 | }, 40 | AssignmentExpression: node => { 41 | throw new Error('AssignmentExpression is not supported: ' + JSON.stringify(node)); 42 | }, 43 | BinaryExpression: binary, 44 | LogicalExpression: binary, 45 | UnaryExpression: node => columns(node.argument), 46 | ConditionalExpression: node => [...columns(node.test), ...columns(node.consequent), ...columns(node.alternate)], 47 | ObjectExpression: node => { 48 | throw new Error('ObjectExpression is not supported: ' + JSON.stringify(node)); 49 | }, 50 | Property: node => { 51 | throw new Error('Property is not supported: ' + JSON.stringify(node)); 52 | }, 53 | 54 | ArrowFunctionExpression: node => { 55 | throw new Error('ArrowFunctionExpression is not supported: ' + JSON.stringify(node)); 56 | }, 57 | FunctionExpression: node => { 58 | throw new Error('FunctionExpression is not supported: ' + JSON.stringify(node)); 59 | }, 60 | FunctionDeclaration: node => { 61 | throw new Error('FunctionDeclaration is not supported: ' + JSON.stringify(node)); 62 | }, 63 | 64 | ArrayPattern: node => { 65 | throw new Error('ArrayPattern is not supported: ' + JSON.stringify(node)); 66 | }, 67 | ObjectPattern: node => { 68 | throw new Error('ObjectPattern is not supported: ' + JSON.stringify(node)); 69 | }, 70 | VariableDeclaration: node => { 71 | throw new Error('VariableDeclaration is not supported: ' + JSON.stringify(node)); 72 | }, 73 | VariableDeclarator: node => { 74 | throw new Error('VariableDeclarator is not supported: ' + JSON.stringify(node)); 75 | }, 76 | SpreadElement: node => { 77 | throw new Error('SpreadElement is not supported: ' + JSON.stringify(node)); 78 | }, 79 | 80 | BlockStatement: node => { 81 | throw new Error('BlockStatement is not supported: ' + JSON.stringify(node)); 82 | }, 83 | BreakStatement: node => { 84 | throw new Error('BreakStatement is not supported: ' + JSON.stringify(node)); 85 | }, 86 | ExpressionStatement: node => columns(node.expression), 87 | IfStatement: node => { 88 | throw new Error('IfStatement is not supported: ' + JSON.stringify(node)); 89 | }, 90 | SwitchStatement: node => { 91 | throw new Error('SwitchStatement is not supported: ' + JSON.stringify(node)); 92 | }, 93 | SwitchCase: node => { 94 | throw new Error('SwitchCase is not supported: ' + JSON.stringify(node)); 95 | }, 96 | ReturnStatement: node => { 97 | throw new Error('ReturnStatement is not supported: ' + JSON.stringify(node)); 98 | }, 99 | Program: node => { 100 | throw new Error('Program is not supported: ' + JSON.stringify(node)); 101 | }, 102 | }; 103 | -------------------------------------------------------------------------------- /src/databases/postgres/visitors/gen-expr.js: -------------------------------------------------------------------------------- 1 | import error from 'arquero/src/util/error'; 2 | 3 | export const ARQUERO_AGGREGATION_FN = ['mean', 'max']; 4 | export const ARQUERO_WINDOW_FN = ['row_number']; 5 | export const ARQUERO_FN = ['random']; 6 | 7 | const ARQUERO_OPS_TO_SQL = { 8 | row_number: 'ROW_NUMBER', 9 | mean: 'AVG', 10 | average: 'AVG', 11 | max: 'MAX', 12 | min: 'MIN', 13 | stdev: 'STDEV', 14 | random: 'RANDOM', 15 | }; 16 | 17 | const BINARY_OPS = { 18 | '===': '=', 19 | '==': '=', 20 | '!==': '<>', 21 | '!=': '<>', 22 | }; 23 | 24 | /** 25 | * 26 | * @param {*} node 27 | * @param {GenExprOpt} opt 28 | */ 29 | export function genExpr(node, opt) { 30 | return visitors[node.type](node, opt); 31 | } 32 | 33 | const binary = (node, opt) => { 34 | if (node.operator === '%') { 35 | return 'MOD(' + genExpr(node.left, opt) + ',' + genExpr(node.right, opt) + ')'; 36 | } 37 | return '(' + genExpr(node.left, opt) + (BINARY_OPS[node.operator] || node.operator) + genExpr(node.right, opt) + ')'; 38 | }; 39 | 40 | const call = (node, opt) => { 41 | if (node.callee.type === 'Function') { 42 | const _args = node.arguments.map(a => genExpr(a, opt)); 43 | switch (node.callee.name) { 44 | case 'equal': 45 | if (_args[0] === 'null') { 46 | return `(${_args[1]} IS NULL)`; 47 | } else if (_args[1] === 'null') { 48 | return `(${_args[0]} IS NULL)`; 49 | } 50 | } 51 | } 52 | 53 | const over = []; 54 | if ( 55 | !opt.withoutOver && 56 | node.callee.type === 'Function' && 57 | [ARQUERO_AGGREGATION_FN, ARQUERO_WINDOW_FN].some(fn => fn.includes(node.callee.name)) 58 | ) { 59 | over.push(' OVER ('); 60 | const toOrder = opt.order && ARQUERO_WINDOW_FN.includes(node.callee.name); 61 | if (opt.partition) { 62 | over.push('PARTITION BY ', opt.partition); 63 | if (toOrder) { 64 | over.push(' '); 65 | } 66 | } 67 | if (toOrder) { 68 | over.push('ORDER BY ', opt.order); 69 | } 70 | over.push(')'); 71 | } 72 | const callee = genExpr(node.callee, opt); 73 | const args = list(node.arguments, opt); 74 | return `(${callee}(${args})${over.join('')})`; 75 | }; 76 | 77 | const list = (array, opt, delim = ',') => { 78 | return array.map(node => genExpr(node, opt)).join(delim); 79 | }; 80 | 81 | const unsuported = node => error(node.type + ' is not supported: ' + JSON.stringify(node)); 82 | 83 | const visitors = { 84 | Column: (node, opt) => { 85 | if (opt && 'index' in opt) throw new Error('row is not supported'); 86 | return `${node.table && opt.tables ? opt.tables[node.table - 1] + '.' : ''}${node.name}`; 87 | }, 88 | Constant: node => node.raw, 89 | Function: node => ARQUERO_OPS_TO_SQL[node.name], 90 | Parameter: unsuported, 91 | OpLookup: unsuported, 92 | Literal: node => node.raw, 93 | Identifier: node => node.name, 94 | TemplateLiteral: (node, opt) => { 95 | const {quasis, expressions} = node; 96 | const n = expressions.length; 97 | let t = '"' + quasis[0].value.raw + '"'; 98 | for (let i = 0; i < n; ) { 99 | t += ', ' + genExpr(expressions[i], opt) + ', "' + quasis[++i].value.raw + '"'; 100 | } 101 | return 'CONCAT(' + t + ')'; 102 | }, 103 | MemberExpression: unsuported, 104 | CallExpression: call, 105 | NewExpression: unsuported, 106 | ArrayExpression: unsuported, 107 | AssignmentExpression: unsuported, 108 | BinaryExpression: binary, 109 | LogicalExpression: binary, 110 | UnaryExpression: (node, opt) => { 111 | return '(' + node.operator + genExpr(node.argument, opt) + ')'; 112 | }, 113 | ConditionalExpression: (node, opt) => { 114 | return ( 115 | '(CASE WHEN ' + 116 | genExpr(node.test, opt) + 117 | ' THEN ' + 118 | genExpr(node.consequent, opt) + 119 | ' ELSE ' + 120 | genExpr(node.alternate, opt) + 121 | ' END)' 122 | ); 123 | }, 124 | ObjectExpression: unsuported, 125 | Property: unsuported, 126 | 127 | ArrowFunctionExpression: unsuported, 128 | FunctionExpression: unsuported, 129 | FunctionDeclaration: unsuported, 130 | 131 | ArrayPattern: unsuported, 132 | ObjectPattern: unsuported, 133 | VariableDeclaration: unsuported, 134 | VariableDeclarator: unsuported, 135 | SpreadElement: unsuported, 136 | 137 | BlockStatement: unsuported, 138 | BreakStatement: unsuported, 139 | ExpressionStatement: (node, opt) => { 140 | return genExpr(node.expression, opt); 141 | }, 142 | IfStatement: unsuported, 143 | SwitchStatement: unsuported, 144 | SwitchCase: unsuported, 145 | ReturnStatement: unsuported, 146 | Program: unsuported, 147 | }; 148 | 149 | /** 150 | * @typedef {object} GenExprOpt 151 | * @prop {string} [partition] 152 | * @prop {string} [order] 153 | * @prop {string[]} [tables] 154 | * @prop {boolean} [withoutOver] 155 | */ 156 | -------------------------------------------------------------------------------- /src/databases/postgres/visitors/has-function.js: -------------------------------------------------------------------------------- 1 | import error from 'arquero/src/util/error'; 2 | 3 | /** 4 | * 5 | * @param {*} node 6 | * @param {string[]} fns 7 | */ 8 | export default function hasFunction(node, fns) { 9 | return visitors[node.type](node, fns); 10 | } 11 | 12 | const binary = (node, fns) => { 13 | return hasFunction(node.left, fns) || hasFunction(node.right, fns); 14 | }; 15 | 16 | const call = (node, fns) => { 17 | return hasFunction(node.callee, fns) || list(node.arguments, fns); 18 | }; 19 | 20 | const list = (array, fns) => { 21 | return array.some(node => hasFunction(node, fns)); 22 | }; 23 | 24 | const unsuported = node => error(node.type + ' is not supported: ' + JSON.stringify(node)); 25 | 26 | const visitors = { 27 | Column: () => false, 28 | Constant: () => false, 29 | Function: (node, fns) => fns.includes(node.name), 30 | Parameter: unsuported, 31 | OpLookup: unsuported, 32 | Literal: () => false, 33 | Identifier: () => false, 34 | TemplateLiteral: (node, fns) => node.expressions.some(e => hasFunction(e, fns)), 35 | MemberExpression: unsuported, 36 | CallExpression: call, 37 | NewExpression: unsuported, 38 | ArrayExpression: unsuported, 39 | AssignmentExpression: unsuported, 40 | BinaryExpression: binary, 41 | LogicalExpression: binary, 42 | UnaryExpression: (node, fns) => hasFunction(node.argument, fns), 43 | ConditionalExpression: (node, fns) => 44 | hasFunction(node.test, fns) || hasFunction(node.consequent, fns) || hasFunction(node.alternate, fns), 45 | ObjectExpression: unsuported, 46 | Property: unsuported, 47 | ArrowFunctionExpression: unsuported, 48 | FunctionExpression: unsuported, 49 | FunctionDeclaration: unsuported, 50 | ArrayPattern: unsuported, 51 | ObjectPattern: unsuported, 52 | VariableDeclaration: unsuported, 53 | VariableDeclarator: unsuported, 54 | SpreadElement: unsuported, 55 | BlockStatement: unsuported, 56 | BreakStatement: unsuported, 57 | ExpressionStatement: (node, fns) => hasFunction(node.expression, fns), 58 | IfStatement: unsuported, 59 | SwitchStatement: unsuported, 60 | SwitchCase: unsuported, 61 | ReturnStatement: unsuported, 62 | Program: unsuported, 63 | }; 64 | -------------------------------------------------------------------------------- /src/databases/table-view.js: -------------------------------------------------------------------------------- 1 | import {internal} from 'arquero'; 2 | 3 | export class TableView extends internal.Transformable { 4 | /** 5 | * @typedef {object} ObjectsOptions 6 | * @property {number} [limit=Infinity] 7 | * @property {number} [offset=0] 8 | * @property {import('../table/transformable').Select} [columns] 9 | * @property {'map'|'entries'|'object'|boolean} [grouped=false] 10 | */ 11 | 12 | /** 13 | * @param {ObjectsOptions} [options] 14 | * @returns {object[]} 15 | */ 16 | // eslint-disable-next-line no-unused-vars 17 | async objects(options) { 18 | throw new Error('objects not supported'); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/db-table.js: -------------------------------------------------------------------------------- 1 | import * as aq from 'arquero'; 2 | import aqVerbs from 'arquero/src/verbs'; 3 | 4 | export class DBTable extends aq.internal.Transformable { 5 | /** 6 | * @param {Promise} tableView 7 | */ 8 | constructor(tableView) { 9 | super({}); 10 | 11 | /** @type {Promise} */ 12 | this._tableView = tableView; 13 | } 14 | 15 | /** 16 | * @returns {Promise} 17 | */ 18 | async toArquero() { 19 | const results = await this.objects(); 20 | return results && aq.from(results); 21 | } 22 | 23 | /** 24 | * @typedef {object} PrintOptions 25 | * @property {number} [limit=Infinity] 26 | * @property {number} [offset=0] 27 | * @property {import('arquero/src/table/transformable').Select} [columns] 28 | */ 29 | 30 | /** 31 | * 32 | * @param {PrintOptions | number} options 33 | */ 34 | async print(options = {}) { 35 | const table = await this.toArquero(); 36 | table.print(options); 37 | return this; 38 | } 39 | 40 | /** 41 | * @typedef {object} ObjectsOptions 42 | * @property {number} [limit=Infinity] 43 | * @property {number} [offset=0] 44 | * @property {import('../table/transformable').Select} [columns] 45 | * @property {'map'|'entries'|'object'|boolean} [grouped=false] 46 | */ 47 | 48 | /** 49 | * @param {ObjectsOptions} [options] 50 | */ 51 | async objects(options = {}) { 52 | return await this._tableView.then(b => b.objects(options)); 53 | } 54 | 55 | /** 56 | * @param {number} row 57 | */ 58 | async object(row = 0) { 59 | const o = await this.objects({limit: 1, offset: row}); 60 | return o && o[0]; 61 | } 62 | } 63 | 64 | // eslint-disable-next-line no-unused-vars 65 | const {__except, __concat, __intersect, __union, ...verbs} = aqVerbs; 66 | 67 | Object.keys(verbs).forEach(verb => (DBTable.prototype[verb] = verbFactory(verb))); 68 | 69 | ['concat', 'intersect', 'except', 'union'] 70 | .map(verb => '__' + verb) 71 | .forEach(verb => (DBTable.prototype[verb] = verbWithOthersFactory(verb))); 72 | 73 | /** 74 | * @param {string} verb 75 | */ 76 | function verbFactory(verb) { 77 | /** 78 | * @param {DBTable} table 79 | * @param {...any} params 80 | */ 81 | function fn(table, ...params) { 82 | const pparams = params.map(param => { 83 | if (param instanceof DBTable) { 84 | return param._tableView; 85 | } else { 86 | return Promise.resolve(param); 87 | } 88 | }); 89 | const pbuilder = Promise.all([table._tableView, ...pparams]).then(([builder, ...resolves]) => 90 | builder[verb](builder, ...resolves), 91 | ); 92 | return new DBTable(pbuilder); 93 | } 94 | 95 | return fn; 96 | } 97 | 98 | /** 99 | * @param {string} verb 100 | */ 101 | function verbWithOthersFactory(verb) { 102 | /** 103 | * @param {DBTable} table 104 | * @param {...any} params 105 | */ 106 | function fn(table, others, ...params) { 107 | const pbuilder = Promise.all([table, ...others].map(o => o._builder)).then(([builder, ...otherBuilders]) => 108 | builder[verb](builder, otherBuilders, ...params), 109 | ); 110 | return new DBTable(pbuilder); 111 | } 112 | 113 | return fn; 114 | } 115 | -------------------------------------------------------------------------------- /src/index-node.js: -------------------------------------------------------------------------------- 1 | export * from './index'; 2 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import {ArqueroDatabase, ArqueroTableView} from './databases/arquero'; 2 | import {PostgresDatabase, PostgresTableView} from './databases/postgres'; 3 | 4 | export {DBTable} from './db-table'; 5 | export {Database, TableView} from './databases'; 6 | 7 | export {genExpr} from './databases/postgres/visitors/gen-expr'; 8 | 9 | export const db = { 10 | Postgres: PostgresDatabase, 11 | Arquero: ArqueroDatabase, 12 | }; 13 | 14 | export const tableView = { 15 | Postgres: PostgresTableView, 16 | Arquero: ArqueroTableView, 17 | }; 18 | -------------------------------------------------------------------------------- /test/code-gen-test.js: -------------------------------------------------------------------------------- 1 | // /** @typedef {import('../src/sql-query').SqlQuery} SqlQuery */ 2 | // /** @typedef {import('arquero').internal.ColumnTable} ColumnTable */ 3 | // import tape from './tape-wrapper'; 4 | // // import {base, base2, base3, group} from './verbs/common'; 5 | // import {all, not, op, startswith, table} from 'arquero'; 6 | // import {types} from 'pg'; 7 | 8 | // import {setupTable2} from './pg-utils'; 9 | 10 | // types.setTypeParser(types.builtins.FLOAT4, val => parseFloat(val)); 11 | // types.setTypeParser(types.builtins.FLOAT8, val => parseFloat(val)); 12 | 13 | // const baseArquero = table({ 14 | // Seattle: [69, 108, 178, 207, 253, 268, 312, 281, 221, 142, 72, 52], 15 | // Chicago: [135, 136, 187, 215, 281, 311, 318, 283, 226, 193, 113, 106], 16 | // San_Francisco: [165, 182, 251, 281, 314, 330, 300, 272, 267, 243, 189, 156], 17 | // }); 18 | // const baseSql = await setupTable2(baseArquero, 'base'); 19 | 20 | // const group = base => base.groupby({a: d => d.Seattle % 10}); 21 | 22 | // const bases = [baseSql, baseArquero]; 23 | // const groups = bases.map(group); 24 | // const [groupSql, groupArquero] = groups; 25 | 26 | // const base1 = bases.map(b => b.filter(d => d.Seattle > 150)); 27 | // const [baseSql1, baseArquero1] = base1; 28 | // const base2 = bases.map(b => b.filter(d => d.Seattle < 200)); 29 | // const [baseSql2, baseArquero2] = base2; 30 | 31 | // /** 32 | // * 33 | // * @param {object} t 34 | // * @param {SqlQuery} actual 35 | // * @param {ColumnTable} expected 36 | // * @param {string} message 37 | // * @param {*} [client] 38 | // */ 39 | // function tableEqual(t, actual, expected, message, client) { 40 | // if (!client) { 41 | // client = connectClient(); 42 | // } 43 | 44 | // const columns = expected.columnNames(); 45 | // const _actual = {}; 46 | // const _expected = {}; 47 | // const expectedData = expected.reify()._data; 48 | // columns.forEach(c => { 49 | // const _c = c.toLowerCase(); 50 | // _actual[_c] = []; 51 | // _expected[_c] = expectedData[c].data; 52 | // }); 53 | // client.querySync(actual.toSql()).forEach(r => { 54 | // Object.entries(r).forEach(([c, v], i) => { 55 | // if (columns[i].toLowerCase() !== c.toLowerCase()) { 56 | // t.fail(`incorrect column order: expecting ${columns[i]}, received ${c}`); 57 | // } 58 | // v = typeof v === 'string' ? parseFloat(v) : v; 59 | // v = v === null ? undefined : v; 60 | // _actual[c].push(v); 61 | // }); 62 | // }); 63 | // t.deepEqual(_actual, _expected, message); 64 | // } 65 | 66 | // // TODO: PostgreSQL does not have CONCAT 67 | // ['intersect', 'except', 'union'].map(v => { 68 | // tape('code-gen: ' + v, t => { 69 | // const client = connectClient(); 70 | // tableEqual( 71 | // t, 72 | // baseSql1[v](baseSql2).orderby('Seattle'), 73 | // baseArquero1[v](baseArquero2).orderby('Seattle'), 74 | // 'basic ' + v, 75 | // client, 76 | // ); 77 | // tableEqual( 78 | // t, 79 | // groupSql[v](baseSql2).orderby('Seattle'), 80 | // groupArquero[v](baseArquero2).orderby('Seattle'), 81 | // 'ungroup before ' + v, 82 | // client, 83 | // ); 84 | // client.end(); 85 | // t.end(); 86 | // }); 87 | // }); 88 | 89 | // tape('code-gen: dedupe', t => { 90 | // const client = connectClient(); 91 | // const query = base => 92 | // base.orderby('Seattle').dedupe({ 93 | // col1: d => d.Seattle % 10, 94 | // }); 95 | // tableEqual(t, ...bases.map(query), 'basic dedupe', client); 96 | 97 | // client.end(); 98 | // t.end(); 99 | // }); 100 | 101 | // tape('code-gen: derive', t => { 102 | // const client = connectClient(); 103 | // const query = base => 104 | // base.derive({ 105 | // col1: d => d.Seattle + d.Chicago, 106 | // col2: d => op.mean(d.Seattle), 107 | // col3: () => op.row_number(), 108 | // }); 109 | // tableEqual(t, ...bases.map(query), 'basic derive', client); 110 | 111 | // client.end(); 112 | // t.end(); 113 | // }); 114 | 115 | // tape('code-gen: filter', t => { 116 | // const client = connectClient(); 117 | // const query = base => base.filter(d => d.Seattle > 200); 118 | // tableEqual(t, ...bases.map(query), 'basic filter', client); 119 | // tableEqual(t, ...groups.map(query), 'basic filter on grouped query', client); 120 | 121 | // const query2 = base => 122 | // base 123 | // .filter(d => op.mean(d.Chicago) > 200) 124 | // // need to order afterward because PostgreSQL does not preserve original order 125 | // .orderby('Seattle'); 126 | // tableEqual(t, ...bases.map(query2), 'filter with aggregated function', client); 127 | // tableEqual(t, ...groups.map(query2), 'filter with aggregated function on grouped query', client); 128 | // client.end(); 129 | // t.end(); 130 | // }); 131 | 132 | // tape('code-gen: groupby', t => { 133 | // const client = connectClient(); 134 | // const query = base => 135 | // base.derive({ 136 | // col1: d => d.Seattle + d.Chicago, 137 | // }); 138 | // tableEqual(t, ...groups.map(query), 'groupby without aggregate/window derive', client); 139 | 140 | // const query2 = base => query(base).rollup().orderby('a'); 141 | // tableEqual(t, ...groups.map(query2), 'groupby with empty rollup', client); 142 | 143 | // const query3 = base => base.rollup({b: d => op.mean(d.Chicago)}).orderby('a'); 144 | // tableEqual(t, ...groups.map(query3), 'groupby with rollup', client); 145 | 146 | // client.end(); 147 | // t.end(); 148 | // }); 149 | 150 | // tape('code-gen: sample', t => { 151 | // const client = connectClient(); 152 | // const query = baseSql.sample(10).orderby('Seattle'); 153 | // const query2 = query.intersect(baseSql); 154 | 155 | // t.deepEqual(client.querySync(query.toSql()), client.querySync(query2.toSql()), 'sample from existing rows'); 156 | 157 | // client.end(); 158 | // t.end(); 159 | // }); 160 | 161 | // tape('code-gen: orderby', t => { 162 | // const client = connectClient(); 163 | // const query = base => base.orderby('Chicago'); 164 | // tableEqual(t, ...bases.map(query), 'simple order', client); 165 | 166 | // const query2 = base => 167 | // query(base) 168 | // .groupby({key: d => d.Seattle > 200}) 169 | // .derive({col: () => op.row_number()}); 170 | // tableEqual(t, ...bases.map(query2), 'ordering for window function', client); 171 | 172 | // const query3 = base => base.groupby({key: d => d.Seattle > 200}).derive({col: d => op.max(d.Chicago)}); 173 | // const query3Sql = base => query3(query(base)); 174 | // const query3Arquero = base => query3(base).orderby('Chicago'); 175 | // tableEqual(t, query3Sql(baseSql), query3Arquero(baseArquero), 'ordering for aggregation function', client); 176 | 177 | // client.end(); 178 | // t.end(); 179 | // }); 180 | 181 | // tape('code-gen: select', t => { 182 | // const client = connectClient(); 183 | // const query = base => base.select('Chicago', 'Seattle'); 184 | // tableEqual(t, ...bases.map(query), 'simple select', client); 185 | 186 | // const query2 = base => base.select(not('Chicago'), 'Chicago'); 187 | // tableEqual(t, ...bases.map(query2), 'select not', client); 188 | 189 | // const query3 = base => base.select(all()); 190 | // tableEqual(t, ...bases.map(query3), 'select all', client); 191 | 192 | // const query4 = base => base.select(startswith('S')); 193 | // tableEqual(t, ...bases.map(query4), 'start with', client); 194 | 195 | // const query5 = base => base.select('Chicago', 'Seattle', {Seattle: 'Seattle2'}); 196 | // tableEqual(t, ...bases.map(query5), 'select with new name', client); 197 | 198 | // client.end(); 199 | // t.end(); 200 | // }); 201 | 202 | // tape('code-gen: ungroup', t => { 203 | // const client = connectClient(); 204 | // const query = base => base.ungroup().rollup({col: d => op.max(d.Seattle)}); 205 | // tableEqual(t, ...bases.map(query), 'ungroup before rollup', client); 206 | // tableEqual(t, ...groups.map(query), 'ungroup before rollup', client); 207 | 208 | // client.end(); 209 | // t.end(); 210 | // }); 211 | 212 | // tape('code-gen: join', t => { 213 | // const client = connectClient(); 214 | // const chicago = baseSql.select('Seattle', 'Chicago'); 215 | // const sanfrancisco = baseSql.select('Seattle', 'San_Francisco'); 216 | // tableEqual( 217 | // t, 218 | // chicago.join(sanfrancisco, 'Seattle').orderby('Seattle'), 219 | // baseArquero.orderby('Seattle'), 220 | // 'simple join', 221 | // client, 222 | // ); 223 | // tableEqual( 224 | // t, 225 | // chicago.join(sanfrancisco, 'Seattle', [all(), all()]).orderby('Seattle_1'), 226 | // baseArquero 227 | // .derive({Seattle_1: d => d.Seattle, Seattle_2: d => d.Seattle}) 228 | // .select('Seattle_1', 'Chicago', 'Seattle_2', 'San_Francisco') 229 | // .orderby('Seattle_1'), 230 | // 'simple join (with custom output columns)', 231 | // client, 232 | // ); 233 | 234 | // const [chicagoSql, chicagoArquero] = base1.map(base => base.select('Seattle', 'Chicago')); 235 | // const [sfSql, sfArquero] = base2.map(base => base.select('Seattle', 'San_Francisco')); 236 | 237 | // ['inner', 'right', 'left', 'outer'].forEach((joinType, idx) => { 238 | // tableEqual( 239 | // t, 240 | // chicagoSql.join(sfSql, 'Seattle', null, {left: idx >> 1, right: idx & 1}).orderby('Seattle'), 241 | // chicagoArquero.join(sfArquero, 'Seattle', null, {left: idx >> 1, right: idx & 1}).orderby('Seattle'), 242 | // joinType + ' join' + (idx >> 1) + ' ' + (idx & 1), 243 | // client, 244 | // ); 245 | // }); 246 | 247 | // client.end(); 248 | // t.end(); 249 | // }); 250 | 251 | // tape('code-gen', t => { 252 | // // const cg6 = base.join(base3, 'a', [all(), not('e')], {left: true}); 253 | // // console.log(codeGen(cg6)); 254 | 255 | // t.end(); 256 | // }); 257 | -------------------------------------------------------------------------------- /test/databases/postgres-test.js: -------------------------------------------------------------------------------- 1 | import * as aq from 'arquero'; 2 | import tape from '../tape-wrapper'; 3 | import {db} from '../../src/index'; 4 | 5 | const database = process.env.PGDB; 6 | const user = process.env.PGUSER; 7 | const password = process.env.PGPASSWORD; 8 | const host = process.env.PGHOST; 9 | const port = process.env.PGPORT; 10 | 11 | const pg = new db.Postgres({user, host, database, password, port}); 12 | 13 | tape('input table', t => { 14 | t.plan(1); 15 | (async () => { 16 | const aq1 = aq.table({ 17 | a: [1, 2, 3, 4], 18 | b: ['1', '2', '3', '4'], 19 | }); 20 | 21 | const db1 = pg.fromArquero(aq1); 22 | t.deepEqual(await db1.objects(), aq1.objects()); 23 | })(); 24 | }); 25 | -------------------------------------------------------------------------------- /test/optimizer-test.js: -------------------------------------------------------------------------------- 1 | import tape from './tape-wrapper'; 2 | // import {SqlQueryBuilder} from '../src'; 3 | // import {all} from 'arquero'; 4 | // import {copy, toAst} from './sql-table-view/common'; 5 | // import createColumn from '../src/utils/create-column'; 6 | 7 | tape('optimizer', t => { 8 | // const table = new SqlQueryBuilder('table', null, {columns: ['Seattle', 'Chicago', 'New York']}); 9 | 10 | // const t0 = table.select(['Chicago']).filter({condition: d => d.Seattle > 100}); 11 | // const fused_t0 = t0.optimize(); 12 | // t.deepEqual(fused_t0._source._source.name, t0._source._source._source.name, 'should fuse inner most table'); 13 | // fused_t0._source._source = null; 14 | // t0._source._source = null; 15 | // t.deepEqual(copy(fused_t0), copy(t0), 'should not fuse if inner query has execution order higher than the outer one'); 16 | 17 | // const t1 = table.filter({condition: d => d.Seattle > 100}).select(['Chicago']); 18 | // const fused_t1 = t1.optimize(); 19 | // t.equal(fused_t1._source, 'table', 'should have correct inner-most table'); 20 | // t.deepEqual( 21 | // copy(fused_t1._clauses.where), 22 | // copy([toAst(d => d.Seattle > 100, 'condition')]), 23 | // 'should have where on the top-most level', 24 | // ); 25 | // t.deepEqual(fused_t1._clauses.select, [createColumn('Chicago')], 'should have select on the top-most level'); 26 | 27 | // const t2 = table.select([all()]).select(['Seattle', 'Chicago']).select(['Seattle']); 28 | // const fused_t2 = t2.optimize(); 29 | // t.equal(fused_t2._source, 'table', 'should have correct inner-most table'); 30 | // t.deepEqual(fused_t2._clauses.select, [createColumn('Seattle')], 'Should use the outer-most select'); 31 | 32 | // const t3 = table.filter({condition1: d => d.Seattle > 100}).filter({condition2: d => d.Chicago > 100}); 33 | // const fused_t3 = t3.optimize(); 34 | // t.equal(fused_t3._source, 'table', 'should have correct inner-most table'); 35 | // t.deepEqual( 36 | // copy(fused_t3._clauses.where), 37 | // copy([toAst(d => d.Chicago > 100, 'condition2'), toAst(d => d.Seattle > 100, 'condition1')]), 38 | // 'Should use the outer-most select', 39 | // ); 40 | 41 | t.end(); 42 | }); 43 | -------------------------------------------------------------------------------- /test/pg-utils.js: -------------------------------------------------------------------------------- 1 | /** @typedef {import('arquero').internal.ColumnTable} ColumnTable */ 2 | import {PostgresDatabase} from '../src/databases/pg-database'; 3 | 4 | const DATABASE = process.env.PGDB; 5 | const USER = process.env.PGUSER; 6 | const PASS = process.env.PGPASSWORD; 7 | const HOST = process.env.PGHOST; 8 | const PORT = process.env.PGPORT; 9 | 10 | const pg = new PostgresDatabase(USER, HOST, DATABASE, PASS, parseInt(PORT)); 11 | 12 | /** 13 | * @param {import('arquero').internal.Table} table 14 | * @param {string} name 15 | */ 16 | export async function setupTable2(table, name) { 17 | return await pg.fromArquero(table, name); 18 | } 19 | -------------------------------------------------------------------------------- /test/sql-query-test.js: -------------------------------------------------------------------------------- 1 | import tape from './tape-wrapper'; 2 | import {PostgresTableView} from '../src/databases/postgres'; 3 | import {copy} from './verbs/common'; 4 | import verbs from '../src/databases/postgres/verbs'; 5 | import {op} from 'arquero'; 6 | 7 | tape('sql-query: interface', t => { 8 | const table1 = new PostgresTableView('table1', ['Seattle', 'Chicago', 'New York']); 9 | const table2 = new PostgresTableView('table2', ['Seattle', 'Chicago', 'New York']); 10 | const table3 = new PostgresTableView('table3', ['Seattle', 'Chicago', 'New York']); 11 | 12 | let actual, expected; 13 | 14 | actual = table1.concat(table2, table3); 15 | expected = verbs.__concat(table1, [table2, table3]); 16 | t.deepEqual(copy(actual), copy(expected), 'correct interface: concat'); 17 | 18 | actual = table1.except(table2, table3); 19 | expected = verbs.__except(table1, [table2, table3]); 20 | t.deepEqual(copy(actual), copy(expected), 'correct interface: except'); 21 | 22 | actual = table1.intersect(table2, table3); 23 | expected = verbs.__intersect(table1, [table2, table3]); 24 | t.deepEqual(copy(actual), copy(expected), 'correct interface: intersect'); 25 | 26 | actual = table1.union(table2, table3); 27 | expected = verbs.__union(table1, [table2, table3]); 28 | t.deepEqual(copy(actual), copy(expected), 'correct interface: union'); 29 | 30 | actual = table1.count({as: 'c'}); 31 | expected = verbs.__count(table1, {as: 'c'}); 32 | t.deepEqual(copy(actual), copy(expected), 'correct interface: count'); 33 | 34 | actual = table1.dedupe('k1', {k2: d => d.k + 1}); 35 | expected = verbs.__dedupe(table1, ['k1', {k2: d => d.k + 1}]); 36 | t.deepEqual(copy(actual), copy(expected), 'correct interface: dedupe'); 37 | actual = table1.dedupe(['k1'], {k2: d => d.k + 1}); 38 | expected = verbs.__dedupe(table1, ['k1', {k2: d => d.k + 1}]); 39 | t.deepEqual(copy(actual), copy(expected), 'correct interface: dedupe'); 40 | 41 | actual = table1.derive({k3: d => d.k1 + d.k2}); 42 | expected = verbs.__derive(table1, {k3: d => d.k1 + d.k2}); 43 | t.deepEqual(copy(actual), copy(expected), 'correct interface: derive'); 44 | 45 | actual = table1.filter(d => d.k1 + d.k2); 46 | expected = verbs.__filter(table1, d => d.k1 + d.k2); 47 | t.deepEqual(copy(actual), copy(expected), 'correct interface: filter'); 48 | 49 | actual = table1.groupby('k1', {k2: d => d.k + 1}); 50 | expected = verbs.__groupby(table1, ['k1', {k2: d => d.k + 1}]); 51 | t.deepEqual(copy(actual), copy(expected), 'correct interface: groupby'); 52 | actual = table1.groupby(['k1'], {k2: d => d.k + 1}); 53 | expected = verbs.__groupby(table1, ['k1', {k2: d => d.k + 1}]); 54 | t.deepEqual(copy(actual), copy(expected), 'correct interface: groupby'); 55 | 56 | actual = table1.join(table2, ['Seattle', 'Chicago'], [['Chicago'], ['Seattle']], {left: true, suffix: ['_0', '_1']}); 57 | expected = verbs.__join(table1, table2, ['Seattle', 'Chicago'], [['Chicago'], ['Seattle']], { 58 | left: true, 59 | suffix: ['_0', '_1'], 60 | }); 61 | t.deepEqual(copy(actual), copy(expected), 'correct interface: join'); 62 | 63 | actual = table1.orderby(['k1'], d => d.k + 1, {k2: d => d.k + 1}); 64 | expected = verbs.__orderby(table1, ['k1', d => d.k + 1, {k2: d => d.k + 1}]); 65 | t.deepEqual(copy(actual), copy(expected), 'correct interface: orderby'); 66 | 67 | actual = table1.rollup({k3: d => op.mean(d.k1 + d.k2)}); 68 | expected = verbs.__rollup(table1, {k3: d => op.mean(d.k1 + d.k2)}); 69 | t.deepEqual(copy(actual), copy(expected), 'correct interface: rollup'); 70 | 71 | actual = table1.sample(5); 72 | expected = verbs.__sample(table1, 5); 73 | t.deepEqual(copy(actual), copy(expected), 'correct interface: sample'); 74 | 75 | actual = table1.select('Seattle', {Chicago: 'c'}); 76 | expected = verbs.__select(table1, ['Seattle', {Chicago: 'c'}]); 77 | t.deepEqual(copy(actual), copy(expected), 'correct interface: select'); 78 | 79 | actual = table1.ungroup(); 80 | expected = verbs.__ungroup(table1); 81 | t.deepEqual(copy(actual), copy(expected), 'correct interface: ungroup'); 82 | 83 | t.end(); 84 | }); 85 | -------------------------------------------------------------------------------- /test/sql-test/buildSql.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS a1; 2 | 3 | CREATE TABLE a1 ( 4 | Seattle int, 5 | Chicago int, 6 | NewYork int 7 | ); 8 | 9 | INSERT INTO a1 (Seattle, Chicago, NewYork) 10 | VALUES (69, 135, 165); 11 | INSERT INTO a1 (Seattle, Chicago, NewYork) 12 | VALUES (108, 136, 182); 13 | INSERT INTO a1 (Seattle, Chicago, NewYork) 14 | VALUES (178, 187, 251); 15 | INSERT INTO a1 (Seattle, Chicago, NewYork) 16 | VALUES (207, 215, 281); 17 | INSERT INTO a1 (Seattle, Chicago, NewYork) 18 | VALUES (253, 281, 314); 19 | INSERT INTO a1 (Seattle, Chicago, NewYork) 20 | VALUES (268, 311, 330); 21 | INSERT INTO a1 (Seattle, Chicago, NewYork) 22 | VALUES (312, 318, 300); 23 | INSERT INTO a1 (Seattle, Chicago, NewYork) 24 | VALUES (281, 283, 272); 25 | INSERT INTO a1 (Seattle, Chicago, NewYork) 26 | VALUES (221, 226, 267); 27 | INSERT INTO a1 (Seattle, Chicago, NewYork) 28 | VALUES (142,193,243); 29 | INSERT INTO a1 (Seattle, Chicago, NewYork) 30 | VALUES (72, 113, 189); 31 | INSERT INTO a1 (Seattle, Chicago, NewYork) 32 | VALUES (52, 106, 156); -------------------------------------------------------------------------------- /test/tape-wrapper.js: -------------------------------------------------------------------------------- 1 | import tape from 'tape'; 2 | import {db} from '../src/index'; 3 | 4 | const database = process.env.PGDB; 5 | const user = process.env.PGUSER; 6 | const password = process.env.PGPASSWORD; 7 | const host = process.env.PGHOST; 8 | const port = process.env.PGPORT; 9 | 10 | let setupPromise = new Promise(resolve => { 11 | tape('setup', t => { 12 | const pg = new db.Postgres({user, host, database, password, port}); 13 | (async () => { 14 | await pg.query('DROP TABLE IF EXISTS a1'); 15 | await pg.query('CREATE TABLE a1 (Seattle INT, Chicago INT, NewYork INT)'); 16 | await pg.query(` 17 | INSERT INTO a1 (Seattle, Chicago, NewYork) 18 | VALUES 19 | (69, 135, 165), 20 | (108, 136, 182), 21 | (178, 187, 251), 22 | (207, 215, 281), 23 | (253, 281, 314), 24 | (268, 311, 330), 25 | (312, 318, 300), 26 | (281, 283, 272), 27 | (221, 226, 267), 28 | (142,193,243), 29 | (72, 113, 189), 30 | (52, 106, 156) 31 | `); 32 | t.end(); 33 | resolve(); 34 | })(); 35 | }); 36 | }); 37 | 38 | /** 39 | * @param {string} name 40 | * @param {tape.TestCase} cb 41 | */ 42 | export default function (name, cb) { 43 | setupPromise = setupPromise.then(() => tape(name, cb)); 44 | } 45 | -------------------------------------------------------------------------------- /test/to-sql-test.js: -------------------------------------------------------------------------------- 1 | import tape from './tape-wrapper'; 2 | // import {SqlTableView} from '../src'; 3 | // import {internal} from 'arquero'; 4 | // const {Verbs} = internal; 5 | 6 | tape('to-sql : well printed', t => { 7 | // const test = new SqlTableView( 8 | // 'table', 9 | // null, 10 | // // Verbs.select(['d => mean(d.foo)'])}, 11 | // ['Seattle', 'Chicago', 'New York'], 12 | // ); 13 | // const t1 = test.select(Verbs.select(['Chicago'])).filter(Verbs.filter({condition: d => d.Seattle > 100})); 14 | 15 | // t.deepEqual(t1.toSql()); 16 | t.end(); 17 | }); 18 | -------------------------------------------------------------------------------- /test/utils-test.js: -------------------------------------------------------------------------------- 1 | import tape from './tape-wrapper'; 2 | import createColumn from '../src/databases/postgres/utils/create-column'; 3 | 4 | tape('createColumn', t => { 5 | t.deepEqual(createColumn('col1'), {type: 'Column', name: 'col1'}, 'create column correctly'); 6 | t.deepEqual( 7 | createColumn('col1', 'col2'), 8 | {type: 'Column', name: 'col1', as: 'col2'}, 9 | 'create column with new output name correctly', 10 | ); 11 | t.end(); 12 | }); 13 | -------------------------------------------------------------------------------- /test/utils.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @param {string} name 3 | * @param {string[]} expectedWarnMessages 4 | * @param {string} message 5 | * @param {(t: object) => any} fn 6 | * @returns {(t: object) => any} 7 | */ 8 | export function consoleWrapper(name, expectedWarnMessages, message, fn) { 9 | return function (t) { 10 | // eslint-disable-next-line no-console 11 | const consoleFn = console[name]; 12 | const consoleMessages = []; 13 | // eslint-disable-next-line no-console 14 | console[name] = m => consoleMessages.push(m); 15 | 16 | fn(t); 17 | t.deepEquals(consoleMessages, expectedWarnMessages, message); 18 | 19 | // eslint-disable-next-line no-console 20 | console[name] = consoleFn; 21 | 22 | t.end(); 23 | }; 24 | } 25 | -------------------------------------------------------------------------------- /test/verbs/common.js: -------------------------------------------------------------------------------- 1 | import {PostgresTableView} from '../../src/databases/postgres'; 2 | import {internal} from 'arquero'; 3 | 4 | export const {Verbs} = internal; 5 | export const base = new PostgresTableView('base', ['a', 'b', 'c', 'd']); 6 | export const base2 = new PostgresTableView('base2', ['a', 'b', 'c', 'd', 'e']); 7 | export const base3 = new PostgresTableView('base3', ['a', 'b', 'c', 'e']); 8 | // export const noschema = new PostgresTableView('no-schema'); 9 | export const group = base.groupby('a', 'b'); 10 | 11 | /** 12 | * deep copy an object 13 | * @param {object} obj object to copy 14 | * @returns copied of obj 15 | */ 16 | export function copy(obj) { 17 | return JSON.parse(JSON.stringify(objToString(obj))); 18 | } 19 | 20 | function objToString(obj) { 21 | if (Array.isArray(obj)) { 22 | return obj.map(o => objToString(o)); 23 | } else if (typeof obj === 'function') { 24 | return obj.toString(); 25 | } else if (typeof obj === 'object' && obj) { 26 | return Object.entries(obj).reduce((acc, [k, v]) => ((acc[k] = objToString(v)), acc), {}); 27 | } else { 28 | return obj; 29 | } 30 | } 31 | 32 | /** 33 | * convert a 1-table JS expression to AST 34 | * @param {function} expr function expression 35 | * @param {string} [as] result column name 36 | * @returns AST of expr 37 | */ 38 | export function toAst(expr, as) { 39 | const _as = as ? {as} : {}; 40 | return {...copy(Verbs.filter(expr).toAST().criteria), ..._as}; 41 | } 42 | 43 | /** 44 | * convert a 2-table JS expression to AST 45 | * @param {function} expr function expression 46 | * @returns AST of expr 47 | */ 48 | export function twoTableExprToAst(expr) { 49 | return copy(Verbs.join('t', expr, [[], []], {}).toAST().on); 50 | } 51 | 52 | /** 53 | * deep equal tests of all actuals and expected 54 | * @param {object} t tape object 55 | * @param {object[]} actuals list of actual values to compare 56 | * @param {[object, string][]} expecteds list of expected values to compare 57 | */ 58 | export function deepEqualAll(t, actuals, expecteds) { 59 | if (actuals.length !== expecteds.length) { 60 | t.fail('actuals and expecteds should have same length but received ' + `${actuals.length} and ${expecteds.length}`); 61 | } 62 | 63 | expecteds.forEach(([expected, message], idx) => t.deepEqual(copy(actuals[idx]), expected, message)); 64 | } 65 | 66 | /** 67 | * 68 | * @param {object} t 69 | * @param {PostgresTableView} actual 70 | * @param {string[]} expectedClauses 71 | */ 72 | export function onlyContainClsuses(t, actual, expectedClauses) { 73 | t.deepEqual(Object.keys(actual._clauses), expectedClauses, `only have ${expectedClauses.join(' ')} clauses`); 74 | } 75 | -------------------------------------------------------------------------------- /test/verbs/count-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import {op} from 'arquero'; 3 | import {group} from './common'; 4 | 5 | tape('verb: count', t => { 6 | const count1 = group.count({as: 'count_'}); 7 | const rollup1 = group.rollup({count_: () => op.count()}); 8 | 9 | t.deepEqual(count1, rollup1, 'desugar into rollup'); 10 | 11 | const count2 = group.count(); 12 | const rollup2 = group.rollup({count: () => op.count()}); 13 | t.deepEqual(count2, rollup2, 'correct default output name'); 14 | 15 | t.end(); 16 | }); 17 | -------------------------------------------------------------------------------- /test/verbs/derive-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import {base, copy, group, onlyContainClsuses} from './common'; 3 | import createColumn from '../../src/databases/postgres/utils/create-column'; 4 | import {GB_KEY} from '../../src/databases/postgres/verbs/groupby'; 5 | import {consoleWrapper} from '../utils'; 6 | import {op} from 'arquero'; 7 | 8 | tape('verb: derive', t => { 9 | const derive = base.derive({f: d => d.a, g: d => d.a + d.b}); 10 | onlyContainClsuses(t, derive, ['select']); 11 | t.deepEqual( 12 | derive._clauses.select.slice(0, base.columnNames().length), 13 | base.columnNames().map(c => createColumn(c)), 14 | 'derive selects original columns', 15 | ); 16 | t.deepEqual( 17 | copy(derive._clauses.select.slice(base.columnNames().length)), 18 | [ 19 | {type: 'Column', name: 'a', as: 'f'}, 20 | { 21 | type: 'BinaryExpression', 22 | left: {type: 'Column', name: 'a'}, 23 | operator: '+', 24 | right: {type: 'Column', name: 'b'}, 25 | as: 'g', 26 | }, 27 | ], 28 | 'derive correct expressions', 29 | ); 30 | t.deepEqual(derive._columns, ['a', 'b', 'c', 'd', 'f', 'g'], 'correct schema'); 31 | 32 | t.end(); 33 | }); 34 | 35 | tape('verb: derive (overriding column)', t => { 36 | const derive = base.derive({f: d => d.a + 1, a: d => d.b + 2}); 37 | onlyContainClsuses(t, derive, ['select']); 38 | t.deepEqual( 39 | derive._clauses.select.slice(1, base.columnNames().length), 40 | base 41 | .columnNames() 42 | .slice(1) 43 | .map(c => createColumn(c)), 44 | 'derive selects original columns', 45 | ); 46 | t.deepEqual( 47 | copy(derive._clauses.select[0]), 48 | { 49 | type: 'BinaryExpression', 50 | left: {type: 'Column', name: 'b'}, 51 | operator: '+', 52 | right: {type: 'Literal', value: 2, raw: '2'}, 53 | as: 'a', 54 | }, 55 | 'derive correct overriden column', 56 | ); 57 | t.deepEqual( 58 | copy(derive._clauses.select[derive._clauses.select.length - 1]), 59 | { 60 | type: 'BinaryExpression', 61 | left: {type: 'Column', name: 'a'}, 62 | operator: '+', 63 | right: {type: 'Literal', value: 1, raw: '1'}, 64 | as: 'f', 65 | }, 66 | 'derive correct expressions', 67 | ); 68 | t.deepEqual(derive._columns, ['a', 'b', 'c', 'd', 'f'], 'correct schema'); 69 | 70 | t.end(); 71 | }); 72 | 73 | tape( 74 | 'verb: derive (grouped query)', 75 | consoleWrapper( 76 | 'warn', 77 | ['Deriving with group may produce output with different ordering of rows'], 78 | 'warn when derive with group', 79 | t => { 80 | const derive = group.derive({f: d => d.a, g: d => d.a + d.b}); 81 | onlyContainClsuses(t, derive, ['select']); 82 | t.deepEqual(derive._source, group, 'only select from previous query'); 83 | t.deepEqual( 84 | copy(derive._clauses.select), 85 | [ 86 | ...base.columnNames().map(c => createColumn(c)), 87 | {type: 'Column', name: 'a', as: 'f'}, 88 | { 89 | type: 'BinaryExpression', 90 | left: {type: 'Column', name: 'a'}, 91 | operator: '+', 92 | right: {type: 'Column', name: 'b'}, 93 | as: 'g', 94 | }, 95 | ...group._group.map(c => createColumn(GB_KEY(c))), 96 | ], 97 | 'derive groupby columns', 98 | ); 99 | t.deepEqual(derive._columns, ['a', 'b', 'c', 'd', 'f', 'g'], 'correct schema'); 100 | t.deepEqual(derive._group, ['a', 'b'], 'group remains the same'); 101 | }, 102 | ), 103 | ); 104 | 105 | tape( 106 | 'verb: derive (grouped query with window function)', 107 | consoleWrapper( 108 | 'warn', 109 | [ 110 | 'Deriving with group may produce output with different ordering of rows', 111 | 'Deriving with window functions with group and without and explicit ordering may produce different result than Arquero', 112 | ], 113 | 'warn when derive with group', 114 | () => group.derive({f: () => op.row_number()}), 115 | ), 116 | ); 117 | 118 | tape('verb: derive (do not allow window and aggregation function in the same expression)', t => { 119 | t.throws(() => { 120 | base.derive({col: d => op.row_number() + op.mean(d)}); 121 | }, 'Cannot derive an expression containing both an aggregation function and a window fundtion'); 122 | 123 | t.end(); 124 | }); 125 | -------------------------------------------------------------------------------- /test/verbs/filter-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import {op} from 'arquero'; 3 | import {base, copy, group, onlyContainClsuses} from './common'; 4 | import createColumn from '../../src/databases/postgres/utils/create-column'; 5 | 6 | tape('verb: filter', t => { 7 | const filter = base.filter(d => d.a === 3); 8 | onlyContainClsuses(t, filter, ['where']); 9 | t.deepEqual(filter._source, base, 'only filter from previous query'); 10 | t.deepEqual( 11 | copy(filter._clauses.where), 12 | [ 13 | { 14 | type: 'BinaryExpression', 15 | left: {type: 'Column', name: 'a'}, 16 | operator: '===', 17 | right: {type: 'Literal', value: 3, raw: '3'}, 18 | }, 19 | ], 20 | 'correct filter', 21 | ); 22 | t.deepEqual(filter._columns, ['a', 'b', 'c', 'd'], 'filter does not change schema'); 23 | 24 | t.end(); 25 | }); 26 | 27 | tape('verb: filter with aggregate function', t => { 28 | const filter = base.filter(d => op.mean(d.a) === 3); 29 | onlyContainClsuses(t, filter, ['select']); 30 | t.deepEqual( 31 | copy(filter._clauses.select), 32 | filter.columnNames().map(c => createColumn(c)), 33 | 'deselect temp column', 34 | ); 35 | t.deepEqual(filter._columns, ['a', 'b', 'c', 'd'], 'filter does not change schema'); 36 | 37 | const filterFilter = filter._source; 38 | onlyContainClsuses(t, filterFilter, ['where']); 39 | t.deepEqual( 40 | copy(filterFilter._clauses.where), 41 | [{type: 'Column', name: '___arquero_sql_predicate___'}], 42 | 'correct filter', 43 | ); 44 | t.deepEqual( 45 | filterFilter._columns, 46 | ['a', 'b', 'c', 'd', '___arquero_sql_predicate___'], 47 | 'filter does not change schema', 48 | ); 49 | 50 | const filterDerive = filterFilter._source; 51 | onlyContainClsuses(t, filterDerive, ['select']); 52 | t.deepEqual(filterDerive._source, base, 'filtering with aggregated function wraps the previous query with derive'); 53 | t.deepEqual( 54 | copy(filterDerive._clauses.select), 55 | [ 56 | {type: 'Column', name: 'a'}, 57 | {type: 'Column', name: 'b'}, 58 | {type: 'Column', name: 'c'}, 59 | {type: 'Column', name: 'd'}, 60 | { 61 | type: 'BinaryExpression', 62 | left: { 63 | type: 'CallExpression', 64 | callee: {type: 'Function', name: 'mean'}, 65 | arguments: [{type: 'Column', name: 'a'}], 66 | }, 67 | operator: '===', 68 | right: {type: 'Literal', value: 3, raw: '3'}, 69 | as: '___arquero_sql_predicate___', 70 | }, 71 | ], 72 | 'correct filter', 73 | ); 74 | t.deepEqual( 75 | filterDerive._columns, 76 | ['a', 'b', 'c', 'd', '___arquero_sql_predicate___'], 77 | 'filter does not change schema', 78 | ); 79 | 80 | t.end(); 81 | }); 82 | 83 | tape('verb: filter after groupby', t => { 84 | const filter = group.filter(d => d.a === 3); 85 | onlyContainClsuses(t, filter, ['where']); 86 | t.deepEqual(filter._source, group, 'only filter from previous query'); 87 | t.deepEqual( 88 | copy(filter._clauses.where), 89 | [ 90 | { 91 | type: 'BinaryExpression', 92 | left: {type: 'Column', name: 'a'}, 93 | operator: '===', 94 | right: {type: 'Literal', value: 3, raw: '3'}, 95 | }, 96 | ], 97 | 'correct filter', 98 | ); 99 | t.deepEqual(filter._columns, ['a', 'b', 'c', 'd'], 'filter does not change schema'); 100 | t.deepEqual(filter._group, group._group, 'filter does not change group'); 101 | 102 | t.end(); 103 | }); 104 | -------------------------------------------------------------------------------- /test/verbs/groupby-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import createColumn from '../../src/databases/postgres/utils/create-column'; 3 | import {GB_KEY} from '../../src/databases/postgres/verbs/groupby'; 4 | import {base, copy, group, onlyContainClsuses} from './common'; 5 | 6 | tape('verb: groupby', t => { 7 | const groupby = base.groupby('a', 'b'); 8 | onlyContainClsuses(t, groupby, ['select']); 9 | t.deepEqual( 10 | copy(groupby._clauses.select), 11 | [...base.columnNames().map(c => createColumn(c)), ...['a', 'b'].map(c => createColumn(c, GB_KEY(c)))], 12 | 'select includes groupby columns', 13 | ); 14 | t.deepEqual(groupby._group, ['a', 'b'], 'annotate group in SqlQuery object'); 15 | t.deepEqual(groupby.columnNames(), base.columnNames(), 'groupby do not change schema'); 16 | t.equal(groupby._source, base, 'groupby wraps over previous query'); 17 | 18 | t.end(); 19 | }); 20 | 21 | tape('verb: groupby with derived columns', t => { 22 | const groupby = base.groupby('a', {f: d => d.a + d.b}); 23 | onlyContainClsuses(t, groupby, ['select']); 24 | t.deepEqual( 25 | copy(groupby._clauses.select), 26 | [ 27 | ...base.columnNames().map(c => createColumn(c)), 28 | {type: 'Column', name: 'a', as: '___arquero_sql_group_a___'}, 29 | { 30 | type: 'BinaryExpression', 31 | left: {type: 'Column', name: 'a'}, 32 | operator: '+', 33 | right: {type: 'Column', name: 'b'}, 34 | as: '___arquero_sql_group_f___', 35 | }, 36 | ], 37 | 'select includes groupby columns', 38 | ); 39 | t.deepEqual(groupby._group, ['a', 'f'], 'annotate group in SqlQuery object'); 40 | t.deepEqual(groupby.columnNames(), base.columnNames(), 'groupby do not change schema'); 41 | 42 | t.end(); 43 | }); 44 | 45 | tape('verb: groupby on grouped query', t => { 46 | const groupby = group.groupby({f: d => d.a + d.b}); 47 | t.deepEqual(groupby, group.ungroup().groupby({f: d => d.a + d.b}), 'ungroup before grouping again'); 48 | 49 | t.end(); 50 | }); 51 | -------------------------------------------------------------------------------- /test/verbs/join-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import createColumn from '../../src/databases/postgres/utils/create-column'; 3 | import {base, base3, copy, onlyContainClsuses} from './common'; 4 | import {JOIN_TYPES} from '../../src/databases/postgres/verbs/join'; 5 | import {not} from 'arquero'; 6 | 7 | /** 8 | * 9 | * @param {number} table 10 | * @param {string} name 11 | */ 12 | function columnIfNotNull(table, name) { 13 | return { 14 | type: 'ConditionalExpression', 15 | test: { 16 | type: 'CallExpression', 17 | callee: {type: 'Function', name: 'equal'}, 18 | arguments: [createColumn(name, name, table), {type: 'Literal', value: null, raw: 'null'}], 19 | }, 20 | consequent: createColumn(name, name, 3 - table), 21 | alternate: createColumn(name, name, table), 22 | as: name, 23 | }; 24 | } 25 | 26 | JOIN_TYPES.forEach((joinType, idx) => { 27 | const option = {left: idx >> 1, right: idx % 2}; 28 | 29 | tape('verb: join ' + joinType.toLowerCase(), t => { 30 | const join = base.join(base3, 'a', null, option); 31 | onlyContainClsuses(t, join, ['select', 'join']); 32 | t.equal(join._source, base, 'join wraps around the previous query'); 33 | t.equal(join._clauses.join.other, base3, 'other table is in join clause'); 34 | 35 | const join_col = table => [createColumn('a', 'a', table)]; 36 | t.deepEqual( 37 | copy(join._clauses.select), 38 | [ 39 | ...(joinType === 'FULL' ? [columnIfNotNull(1, 'a')] : []), 40 | ...(joinType === 'INNER' || joinType === 'LEFT' ? join_col(1) : []), 41 | createColumn('b', 'b_1', 1), 42 | createColumn('c', 'c_1', 1), 43 | createColumn('d', 'd', 1), 44 | ...(joinType === 'RIGHT' ? join_col(2) : []), 45 | createColumn('b', 'b_2', 2), 46 | createColumn('c', 'c_2', 2), 47 | createColumn('e', 'e', 2), 48 | ], 49 | "select both tables' columns with suffixes", 50 | ); 51 | t.deepEqual( 52 | copy(join._clauses.join.on), 53 | { 54 | type: 'BinaryExpression', 55 | left: {type: 'Column', name: 'a', table: 1}, 56 | operator: '===', 57 | right: {type: 'Column', name: 'a', table: 2}, 58 | }, 59 | 'correct join expression', 60 | ); 61 | t.equal(join._clauses.join.join_type, joinType, 'correct join type'); 62 | 63 | t.end(); 64 | }); 65 | 66 | tape('verb: join ' + joinType.toLowerCase() + ' (with join expression)', t => { 67 | const join = base.join(base3, (a, b) => a.a === b.b + b.c, null, option); 68 | 69 | t.deepEqual( 70 | copy(join._clauses.select), 71 | [ 72 | createColumn('a', 'a_1', 1), 73 | createColumn('b', 'b_1', 1), 74 | createColumn('c', 'c_1', 1), 75 | createColumn('d', 'd', 1), 76 | createColumn('a', 'a_2', 2), 77 | createColumn('b', 'b_2', 2), 78 | createColumn('c', 'c_2', 2), 79 | createColumn('e', 'e', 2), 80 | ], 81 | "select both tables' columns with suffixes", 82 | ); 83 | t.deepEqual( 84 | copy(join._clauses.join.on), 85 | { 86 | type: 'BinaryExpression', 87 | left: {type: 'Column', name: 'a', table: 1}, 88 | operator: '===', 89 | right: { 90 | type: 'BinaryExpression', 91 | left: {type: 'Column', name: 'b', table: 2}, 92 | operator: '+', 93 | right: {type: 'Column', name: 'c', table: 2}, 94 | }, 95 | }, 96 | 'correct join expression', 97 | ); 98 | 99 | t.end(); 100 | }); 101 | 102 | tape('verb: join ' + joinType.toLowerCase() + ' (with all columns)', t => { 103 | const join = base.join(base3, null, null, option); 104 | 105 | const commonColumns = table => [ 106 | createColumn('a', 'a', table), 107 | createColumn('b', 'b', table), 108 | createColumn('c', 'c', table), 109 | ]; 110 | 111 | const outerColumns = [ 112 | columnIfNotNull(1, 'a'), 113 | columnIfNotNull(1, 'b'), 114 | columnIfNotNull(1, 'c'), 115 | createColumn('d', 'd', 1), 116 | createColumn('e', 'e', 2), 117 | ]; 118 | const nonOuterColumns = [ 119 | ...(joinType !== 'RIGHT' ? commonColumns(1) : []), 120 | createColumn('d', 'd', 1), 121 | ...(joinType === 'RIGHT' ? commonColumns(2) : []), 122 | createColumn('e', 'e', 2), 123 | ]; 124 | 125 | t.deepEqual( 126 | copy(join._clauses.select), 127 | joinType === 'FULL' ? outerColumns : nonOuterColumns, 128 | "select both tables' columns with suffixes", 129 | ); 130 | t.deepEqual( 131 | copy(join._clauses.join.on), 132 | { 133 | type: 'LogicalExpression', 134 | left: { 135 | type: 'LogicalExpression', 136 | left: { 137 | type: 'BinaryExpression', 138 | left: {type: 'Column', name: 'a', table: 1}, 139 | operator: '===', 140 | right: {type: 'Column', name: 'a', table: 2}, 141 | }, 142 | operator: '&&', 143 | right: { 144 | type: 'BinaryExpression', 145 | left: {type: 'Column', name: 'b', table: 1}, 146 | operator: '===', 147 | right: {type: 'Column', name: 'b', table: 2}, 148 | }, 149 | }, 150 | operator: '&&', 151 | right: { 152 | type: 'BinaryExpression', 153 | left: {type: 'Column', name: 'c', table: 1}, 154 | operator: '===', 155 | right: {type: 'Column', name: 'c', table: 2}, 156 | }, 157 | }, 158 | 'correct join expression', 159 | ); 160 | 161 | t.end(); 162 | }); 163 | 164 | tape('verb: join ' + joinType.toLowerCase() + ' with custom values', t => { 165 | const join = base.join(base3, 'a', [not('c'), ['e', 'b', 'b', 'e']], option); 166 | 167 | t.deepEqual( 168 | copy(join._clauses.select), 169 | [ 170 | {type: 'Column', name: 'a', table: 1}, 171 | {type: 'Column', name: 'b', table: 1, as: 'b_1'}, 172 | {type: 'Column', name: 'd', table: 1}, 173 | {type: 'Column', name: 'e', table: 2}, 174 | {type: 'Column', name: 'b', table: 2, as: 'b_2'}, 175 | ], 176 | "select both tables' columns with suffixes", 177 | ); 178 | t.deepEqual( 179 | copy(join._clauses.join.on), 180 | { 181 | type: 'BinaryExpression', 182 | left: {type: 'Column', name: 'a', table: 1}, 183 | operator: '===', 184 | right: {type: 'Column', name: 'a', table: 2}, 185 | }, 186 | 'correct join expression', 187 | ); 188 | 189 | t.end(); 190 | }); 191 | }); 192 | -------------------------------------------------------------------------------- /test/verbs/orderby-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import {desc} from 'arquero'; 3 | import {base, copy, onlyContainClsuses} from './common'; 4 | 5 | tape('verb: orderby', t => { 6 | const orderby = base.orderby(desc('a'), d => d.b * 2, desc({k: d => d.a + 3})); 7 | onlyContainClsuses(t, orderby, []); 8 | t.deepEqual(orderby._source, base, 'orderby wraps around the previous query'); 9 | t.deepEqual( 10 | copy(orderby._order.exprs), 11 | [ 12 | {type: 'Column', name: 'a'}, 13 | { 14 | type: 'BinaryExpression', 15 | left: {type: 'Column', name: 'b'}, 16 | operator: '*', 17 | right: {type: 'Literal', value: 2, raw: '2'}, 18 | }, 19 | { 20 | type: 'BinaryExpression', 21 | left: {type: 'Column', name: 'a'}, 22 | operator: '+', 23 | right: {type: 'Literal', value: 3, raw: '3'}, 24 | }, 25 | ], 26 | 'orderby annotate SqlQuery object with comparators', 27 | ); 28 | t.deepEqual(orderby._order.descs, [true, false, true], 'orderby annotate SqlQuery object with order directions'); 29 | 30 | t.end(); 31 | }); 32 | 33 | tape('verb: orderby before other query', t => { 34 | const orderby = base.orderby(desc('a')).filter(d => d.a === 1); 35 | t.deepEqual( 36 | copy(orderby._order.exprs), 37 | [{type: 'Column', name: 'a'}], 38 | 'orderby annotate SqlQuery object with comparators', 39 | ); 40 | t.deepEqual(orderby._order.descs, [true], 'orderby annotate SqlQuery object with order directions'); 41 | 42 | t.end(); 43 | }); 44 | -------------------------------------------------------------------------------- /test/verbs/rollup-test.js: -------------------------------------------------------------------------------- 1 | import {op} from 'arquero'; 2 | import tape from '../tape-wrapper'; 3 | import createColumn from '../../src/databases/postgres/utils/create-column'; 4 | import {GB_KEY} from '../../src/databases/postgres/verbs/groupby'; 5 | import {base, copy, group, onlyContainClsuses} from './common'; 6 | 7 | tape('verb: rollup', t => { 8 | const rollup = group.rollup({k: d => op.mean(d.a)}); 9 | onlyContainClsuses(t, rollup, ['select', 'groupby']); 10 | t.deepEqual(rollup._source, group, 'rollup wraps around the previous query'); 11 | t.deepEqual( 12 | copy(rollup._clauses.select), 13 | [ 14 | ...group._group.map(c => createColumn(GB_KEY(c), c)), 15 | { 16 | type: 'CallExpression', 17 | callee: {type: 'Function', name: 'mean'}, 18 | arguments: [{type: 'Column', name: 'a'}], 19 | as: 'k', 20 | }, 21 | ], 22 | 'rollup selects groupby keys and rollup expressions', 23 | ); 24 | t.deepEqual(rollup._columns, [...group._group, 'k'], 'correct schema'); 25 | t.equal(rollup._group, null, 'query is no longer grouped'); 26 | 27 | t.end(); 28 | }); 29 | 30 | tape('verb: rollup without groupby', t => { 31 | const rollup = base.rollup({k: d => op.mean(d.a)}); 32 | onlyContainClsuses(t, rollup, ['select', 'groupby']); 33 | t.deepEqual(rollup._source, base, 'rollup wraps around the previous query'); 34 | t.deepEqual( 35 | copy(rollup._clauses.select), 36 | [ 37 | { 38 | type: 'CallExpression', 39 | callee: {type: 'Function', name: 'mean'}, 40 | arguments: [{type: 'Column', name: 'a'}], 41 | as: 'k', 42 | }, 43 | ], 44 | 'rollup selects groupby keys and rollup expressions', 45 | ); 46 | t.equal(rollup._clauses.groupby, true, 'group all into 1 row'); 47 | t.deepEqual(rollup._columns, ['k'], 'correct schema'); 48 | 49 | t.end(); 50 | }); 51 | 52 | tape('verb: rollup (do not allow window function)', t => { 53 | t.throws(() => { 54 | base.rollup({col: () => op.row_number()}); 55 | }, 'Cannot rollup an expression containing a window fundtion'); 56 | 57 | t.end(); 58 | }); 59 | -------------------------------------------------------------------------------- /test/verbs/select-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import {not} from 'arquero'; 3 | import {base, copy, group, onlyContainClsuses} from './common'; 4 | 5 | tape('verb: select', t => { 6 | const select = base.select(1, 'd'); 7 | onlyContainClsuses(t, select, ['select']); 8 | t.deepEqual( 9 | copy(select._clauses.select), 10 | [ 11 | {type: 'Column', name: 'b'}, 12 | {type: 'Column', name: 'd'}, 13 | ], 14 | 'correct selection with number column and name column', 15 | ); 16 | t.deepEqual(select._source, base, 'select should wrap the previous query'); 17 | t.deepEqual(select.columnNames(), ['b', 'd'], 'select produces correct schema'); 18 | 19 | t.end(); 20 | }); 21 | 22 | tape('verb: select with selection function', t => { 23 | const select = base.select('d', not('b')); 24 | onlyContainClsuses(t, select, ['select']); 25 | t.deepEqual( 26 | copy(select._clauses.select), 27 | [ 28 | {type: 'Column', name: 'd'}, 29 | {type: 'Column', name: 'a'}, 30 | {type: 'Column', name: 'c'}, 31 | ], 32 | 'correct selection with selection function', 33 | ); 34 | t.deepEqual(select.columnNames(), ['d', 'a', 'c'], 'select produces correct schema'); 35 | 36 | t.end(); 37 | }); 38 | 39 | tape('verb: select with grouped query', t => { 40 | const select = group.select('d', not('b')); 41 | onlyContainClsuses(t, select, ['select']); 42 | t.deepEqual( 43 | copy(select._clauses.select), 44 | [ 45 | {type: 'Column', name: 'd'}, 46 | {type: 'Column', name: 'a'}, 47 | {type: 'Column', name: 'c'}, 48 | {type: 'Column', name: '___arquero_sql_group_a___'}, 49 | {type: 'Column', name: '___arquero_sql_group_b___'}, 50 | ], 51 | 'selection includes groupby columns', 52 | ); 53 | t.deepEqual(select.columnNames(), ['d', 'a', 'c'], 'select produces correct schema without groupby columns'); 54 | t.deepEqual(select._group, group._group, 'the new SqlQuery object is annotated with groupby keys'), t.end(); 55 | }); 56 | -------------------------------------------------------------------------------- /test/verbs/set-verbs-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import createColumn from '../../src/databases/postgres/utils/create-column'; 3 | import {base, base2, group} from './common'; 4 | 5 | ['concat', 'except', 'intersect', 'union'].forEach(verb => { 6 | tape('verb: ' + verb, t => { 7 | const sv1 = base[verb](group); 8 | t.equal(sv1._source, base, 'store inner table'); 9 | t.equal(sv1._clauses[verb].length, 1, `${verb} correctly`); 10 | t.deepEqual(sv1._clauses[verb][0], group.ungroup(), `${verb} correctly`); 11 | t.deepEqual(sv1._columns, base._columns, 'correct schema'); 12 | t.deepEqual( 13 | sv1._clauses.select, 14 | base.columnNames().map(c => createColumn(c)), 15 | "select first table's columns", 16 | ); 17 | 18 | const sv2 = base[verb](base2, group); 19 | t.equal(2, sv2._clauses[verb].length, `${verb} correctly`); 20 | t.deepEqual(sv2._clauses[verb][0], base2, `${verb} correctly`); 21 | t.deepEqual(sv2._clauses[verb][1], group.ungroup(), `${verb} correctly`); 22 | 23 | t.end(); 24 | }); 25 | }); 26 | -------------------------------------------------------------------------------- /test/verbs/ungroup-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import {base, group} from './common'; 3 | 4 | tape('verb: ungroup', t => { 5 | const ungroup = group.ungroup(); 6 | t.deepEqual(ungroup._source, group, 'ungroup wraps around the previous query'); 7 | t.notOk(ungroup._group, 'should not contain group'); 8 | 9 | t.end(); 10 | }); 11 | 12 | tape('verb: ungroup a query without group', t => { 13 | const ungroup = base.ungroup(); 14 | t.equal(ungroup, base, 'does not make any change'); 15 | 16 | t.end(); 17 | }); 18 | -------------------------------------------------------------------------------- /test/verbs/unorder-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import {base} from './common'; 3 | 4 | tape('verb: unorder', t => { 5 | const unorder = base.orderby('a').unorder(); 6 | t.deepEqual(unorder._source, base.orderby('a'), 'ungroup wraps around the previous query'); 7 | t.notOk(unorder._order, 'should not contain order'); 8 | 9 | t.end(); 10 | }); 11 | 12 | tape('verb: unorder a query without order', t => { 13 | const unorder = base.unorder(); 14 | t.equal(unorder, base, 'does not make any change'); 15 | 16 | t.end(); 17 | }); 18 | -------------------------------------------------------------------------------- /test/visitors/gen-expr-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import {genExpr} from '../../src/databases/postgres/visitors/gen-expr'; 3 | import {internal, op} from 'arquero'; 4 | 5 | const {Verbs} = internal; 6 | 7 | tape('gen-expr: 1 table expression', t => { 8 | const exprs = Verbs.derive({ 9 | constant: () => 1 + 1, 10 | column1: d => d.a * d.b, 11 | column2: d => d.a * (d.b + 3), 12 | agg1: d => op.mean(d.a), 13 | row_num: () => op.row_number(), 14 | random: () => op.random(), 15 | }).toAST().values; 16 | 17 | const common_exprs = ['(1+1)', '(a*b)', '(a*(b+3))']; 18 | 19 | t.deepEqual( 20 | exprs.map(expr => genExpr(expr, {})), 21 | [...common_exprs, '(AVG(a) OVER ())', '(ROW_NUMBER() OVER ())', '(RANDOM())'], 22 | 'should generate expression correctly', 23 | ); 24 | 25 | t.deepEqual( 26 | exprs.map(expr => genExpr(expr, {partition: 'a,b'})), 27 | [...common_exprs, '(AVG(a) OVER (PARTITION BY a,b))', '(ROW_NUMBER() OVER (PARTITION BY a,b))', '(RANDOM())'], 28 | 'should generate expression correctly: with partition', 29 | ); 30 | 31 | t.deepEqual( 32 | exprs.map(expr => genExpr(expr, {order: 'c,d'})), 33 | [...common_exprs, '(AVG(a) OVER ())', '(ROW_NUMBER() OVER (ORDER BY c,d))', '(RANDOM())'], 34 | 'should generate expression correctly: with order', 35 | ); 36 | 37 | t.deepEqual( 38 | exprs.map(expr => genExpr(expr, {partition: 'a,b', order: 'c,d'})), 39 | [ 40 | ...common_exprs, 41 | '(AVG(a) OVER (PARTITION BY a,b))', 42 | '(ROW_NUMBER() OVER (PARTITION BY a,b ORDER BY c,d))', 43 | '(RANDOM())', 44 | ], 45 | 'should generate expression correctly: with partition and order', 46 | ); 47 | 48 | t.end(); 49 | }); 50 | 51 | tape('gen-expr: 2 tables expression', t => { 52 | const expr = Verbs.join('t2', (a, b) => a.k1 === b.k2, ['_1', '_2']).toAST(); 53 | t.deepEqual( 54 | genExpr(expr.on, {tables: ['t1', 't2']}), 55 | '(t1.k1=t2.k2)', 56 | "should generate expression with tables' alias correctly", 57 | ); 58 | 59 | t.end(); 60 | }); 61 | -------------------------------------------------------------------------------- /test/visitors/has-function-test.js: -------------------------------------------------------------------------------- 1 | import tape from '../tape-wrapper'; 2 | import {internal, op} from 'arquero'; 3 | import hasFunction from '../../src/databases/postgres/visitors/has-function'; 4 | import {ARQUERO_AGGREGATION_FN, ARQUERO_WINDOW_FN} from '../../src/databases/postgres/visitors/gen-expr'; 5 | 6 | const {Verbs} = internal; 7 | 8 | tape('has-function', t => { 9 | const exprs = Verbs.derive({ 10 | constant: () => 1 + 1, 11 | column1: d => d.a * d.b, 12 | column2: d => d.a * (d.b + 3), 13 | agg1: d => op.mean(d.a), 14 | row_num: () => op.row_number(), 15 | }).toAST().values; 16 | 17 | t.deepEqual( 18 | exprs.map(expr => hasFunction(expr, ARQUERO_AGGREGATION_FN)), 19 | [false, false, false, true, false], 20 | 'can detect aggregation operations', 21 | ); 22 | 23 | t.deepEqual( 24 | exprs.map(expr => hasFunction(expr, ARQUERO_WINDOW_FN)), 25 | [false, false, false, false, true], 26 | 'can detect window operations', 27 | ); 28 | 29 | t.end(); 30 | }); 31 | --------------------------------------------------------------------------------