├── docs ├── _config.yml ├── performance.md ├── evaluation.md ├── github.md ├── internals.md ├── events.md └── index.md ├── dist └── sql-wasm.wasm ├── testEndToEnd ├── data │ ├── cache.sqlite │ ├── students.sqlite │ ├── flights.small.sqlite │ └── sensors_10000.sqlite ├── flightTest.ts ├── README.md ├── tsconfig.json ├── diel │ ├── test.diel │ ├── students_range.diel │ ├── students.diel │ ├── student_range_multi.diel │ ├── cache.diel │ ├── simple.diel │ ├── sensors.diel │ └── flights-remote.diel ├── perfEval.ts ├── index.html ├── index.ts ├── cacheTest.ts ├── sensorTest.ts ├── testComplexCaching.ts └── studentTest.ts ├── tests ├── testTypes.ts ├── readme.md ├── tsconfig.json ├── compilerTests │ ├── testOperators.ts │ ├── testMaterializationRuntime.ts │ ├── assertDecomposition.ts │ ├── assertNormalization.ts │ ├── assertTypes.ts │ ├── testStarExpantion.ts │ ├── testAsyncPolicy.ts │ ├── testDependency.ts │ ├── testSyntaxSugar.ts │ ├── testViewConstraints.ts │ ├── testConstraintMaterializedView.ts │ └── testMaterialization.ts ├── parserTests │ ├── functionTest.ts │ ├── basicOperatorsTest.ts │ └── constraintsTest.ts ├── testEventTableCacheCreation.ts ├── testHelper.ts ├── sqlCodeGenTest.ts ├── index.ts └── unitTest.ts ├── src ├── util │ ├── dielUdfs.ts │ ├── messages.ts │ └── dielUtils.ts ├── tsconfig.json ├── compiler │ ├── codegen │ │ ├── SqlAstGetters.ts │ │ └── staticSql.ts │ ├── passes │ │ ├── passesHelper.ts │ │ ├── normalizeConstraints.ts │ │ ├── normalizeAlias.ts │ │ ├── syntaxSugar.ts │ │ ├── applyTemplate.ts │ │ ├── inferType.ts │ │ ├── materialization.ts │ │ ├── generateViewConstraints.ts │ │ └── dependency.ts │ ├── DielAstVisitors.ts │ ├── compiler.ts │ └── DielAstGetters.ts ├── parser │ ├── visitorHelper.ts │ ├── customError.ts │ └── sqlAstTypes.ts ├── index.ts └── runtime │ ├── runtimeTypes.ts │ ├── runtimeHelper.ts │ ├── ConnectionWrapper.ts │ └── asyncPolicies.ts ├── .travis.yml ├── .gitignore ├── codecov.yml ├── tsconfig.json ├── rollup.config.js ├── package.json ├── webpack.config.js ├── README.md └── tslint.json /docs/_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-minimal -------------------------------------------------------------------------------- /dist/sql-wasm.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yifanwu/diel/HEAD/dist/sql-wasm.wasm -------------------------------------------------------------------------------- /testEndToEnd/data/cache.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yifanwu/diel/HEAD/testEndToEnd/data/cache.sqlite -------------------------------------------------------------------------------- /testEndToEnd/data/students.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yifanwu/diel/HEAD/testEndToEnd/data/students.sqlite -------------------------------------------------------------------------------- /tests/testTypes.ts: -------------------------------------------------------------------------------- 1 | export interface TestLogger { 2 | error: (m: string, o?: any) => void; 3 | pass: () => void; 4 | } -------------------------------------------------------------------------------- /testEndToEnd/data/flights.small.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yifanwu/diel/HEAD/testEndToEnd/data/flights.small.sqlite -------------------------------------------------------------------------------- /testEndToEnd/data/sensors_10000.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yifanwu/diel/HEAD/testEndToEnd/data/sensors_10000.sqlite -------------------------------------------------------------------------------- /testEndToEnd/flightTest.ts: -------------------------------------------------------------------------------- 1 | // const dielFiles = [path.resolve(__dirname, "../../testEndToEnd/diel/flights-remote.diel")]; 2 | -------------------------------------------------------------------------------- /tests/readme.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | ## Dev Uses 4 | 5 | Use `index.ts` to change the functions you want to test. 6 | 7 | ## Full tests -------------------------------------------------------------------------------- /testEndToEnd/README.md: -------------------------------------------------------------------------------- 1 | # Instructions for running 2 | 3 | ```bash 4 | npm install 5 | npm run testEndToEnd 6 | ``` 7 | 8 | Then go to `http://localhost:8080/testEndToEnd/` for the test -------------------------------------------------------------------------------- /src/util/dielUdfs.ts: -------------------------------------------------------------------------------- 1 | export function log(msg: string, source: string) { 2 | console.log(`[${source}] ${msg}`); 3 | return 1; 4 | } 5 | 6 | export function timeNow() { 7 | return +new Date(); 8 | } -------------------------------------------------------------------------------- /src/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "..", 3 | "compilerOptions": { 4 | "outDir": "../build", 5 | "rootDir": ".." 6 | }, 7 | "files": ["index.ts"], 8 | "include": ["**/*", "../package.json"] 9 | } 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "8" 4 | - "10" 5 | 6 | before_script: 7 | - npm run setup 8 | - npm run lang 9 | - npm run build 10 | 11 | script: 12 | - npm run test 13 | 14 | after_success: 15 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /tests/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "..", 3 | "files": ["index.ts"], 4 | "compilerOptions": { 5 | "outDir": "../build", 6 | "rootDir": ".." 7 | }, 8 | "references": [ 9 | { 10 | "path": "../src" 11 | } 12 | ], 13 | "include":["**/*"], 14 | } 15 | -------------------------------------------------------------------------------- /testEndToEnd/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "..", 3 | "files": ["index.ts"], 4 | "compilerOptions": { 5 | "outDir": "../build", 6 | "rootDir": ".." 7 | }, 8 | "references": [ 9 | { 10 | "path": "../src" 11 | } 12 | ], 13 | "include":["**/*"], 14 | } 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules/ 3 | build/ 4 | dist/ 5 | DIELParser$* 6 | trash/ 7 | src/parser/grammar/*.ts 8 | src/parser/grammar/*.ts 9 | .antlr 10 | 11 | *.tsbuildinfo 12 | *.tgz 13 | *.java 14 | *.class 15 | *.tokens 16 | *.jar 17 | *tmp.* 18 | *.map 19 | *.d.ts 20 | *.csv 21 | *.txt 22 | *.db 23 | *.scrap 24 | -------------------------------------------------------------------------------- /docs/performance.md: -------------------------------------------------------------------------------- 1 | # Performance Enhancement Techniques 2 | 3 | The workload for DIEL is such that we have a lot of joins, these joins are pretty cheap if materialized immediately, but it seems that SQLite is a bit dumb, so a good way to use DIEL is to create lots of views and DIEL will force materialization (by creating triggers). 4 | -------------------------------------------------------------------------------- /testEndToEnd/diel/test.diel: -------------------------------------------------------------------------------- 1 | create output allOriginAirports AS 2 | select * from allOriginAirportsEvent 3 | constrain check (c > 100); 4 | 5 | create event view allOriginAirportsEvent AS 6 | select 7 | origin, 8 | count() as c, 9 | avg(delay) as avgDelay 10 | from flights 11 | group by origin 12 | order by c 13 | limit 20 14 | constrain check (c > 80); 15 | 16 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | notify: 3 | require_ci_to_pass: true 4 | comment: false 5 | coverage: 6 | precision: 2 7 | range: 8 | - 70.0 9 | - 100.0 10 | round: down 11 | status: 12 | changes: false 13 | patch: true 14 | project: true 15 | parsers: 16 | gcov: 17 | branch_detection: 18 | conditional: true 19 | loop: true 20 | macro: false 21 | method: false 22 | javascript: 23 | enable_partials: false -------------------------------------------------------------------------------- /src/compiler/codegen/SqlAstGetters.ts: -------------------------------------------------------------------------------- 1 | import { SqlRelation, SqlOriginalRelation, SqlAst } from "../../parser/sqlAstTypes"; 2 | import { RelationNameType } from "../../parser/dielAstTypes"; 3 | 4 | export function GetSqlRelationFromAst(ast: SqlAst, rName: RelationNameType): SqlRelation | null { 5 | return ast.relations.find(r => r.rName === rName); 6 | } 7 | 8 | export function GetDynamicRelationsColumns(r: SqlOriginalRelation): string[] { 9 | return r.columns.map(c => c.cName); 10 | } -------------------------------------------------------------------------------- /testEndToEnd/perfEval.ts: -------------------------------------------------------------------------------- 1 | import { testStudentDb } from "./studentTest"; 2 | import { DielRuntime } from "../src"; 3 | 4 | // this file evaluates the performance by stress testing DIEL and reporting the numbers 5 | 6 | // we want to do a minial one that's otherwise super fast, just to see the overhead of DIEL 7 | export function baseLineEval(perf: (diel: DielRuntime) => void) { 8 | testStudentDb(perf); 9 | // print the results 10 | } 11 | 12 | // then a more complex one, and toggle on and off the caching and materialization 13 | 14 | -------------------------------------------------------------------------------- /testEndToEnd/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | DIEL End to End Test 6 | 7 | 8 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /testEndToEnd/index.ts: -------------------------------------------------------------------------------- 1 | import { testStudentDb } from "./studentTest"; 2 | import { testRangeCaching, testMultiTableCaching } from "./testComplexCaching"; 3 | import { baseLineEval } from "./perfEval"; 4 | import { sensorTest } from "./sensorTest"; 5 | import { DielRuntime } from "../src"; 6 | 7 | const perf = (diel: DielRuntime) => { 8 | diel.inspectQueryResult(`select * from __perf`); 9 | diel.downloadDB(1); 10 | diel.ShutDown(); 11 | }; 12 | //sensorTest(perf); 13 | // baseLineEval(perf); 14 | // testMultiTableCaching(); 15 | // testRangeCaching(); 16 | testStudentDb(perf); 17 | -------------------------------------------------------------------------------- /testEndToEnd/diel/students_range.diel: -------------------------------------------------------------------------------- 1 | -- find student whose exams are within a range 2 | create event table score_range (low integer, high integer); 3 | 4 | create view current_score_range as select low, high from latest score_range; 5 | 6 | create event view matching_students AS 7 | select student.first_name 8 | from students natural join exam 9 | join current_score_range 10 | where exam.score < current_score_range.high and exam.score > current_score_range.low; 11 | 12 | -- find the latest requested 13 | create output current_matching_students AS 14 | select first_name 15 | from matching_students 16 | where request_timestep = (select max(timestep) from score_range); -------------------------------------------------------------------------------- /testEndToEnd/diel/students.diel: -------------------------------------------------------------------------------- 1 | -- remote table students(first_name text, id integer) 2 | -- remote table exam(id integer, score integer) 3 | 4 | create event table name_choice (first_name text); 5 | 6 | create view latest_name_choice as select first_name from latest name_choice; 7 | 8 | create event view current_student_score 9 | as select students.first_name as student, exam.score as grade 10 | from students natural join exam natural join latest_name_choice; 11 | 12 | create output grade_result as 13 | select student, grade 14 | from current_student_score 15 | where request_timestep = ( 16 | select max(request_timestep) 17 | from current_student_score 18 | ); -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es6", 4 | "module": "commonjs", 5 | "lib": ["es6", "dom"], // hm what's with the dom 6 | 7 | "declaration": true, 8 | "declarationMap": true, 9 | "sourceMap": true, 10 | // "strictNullChecks": true, // occastionally turn on to improve code quality.... 11 | 12 | "noImplicitAny": true, 13 | "noImplicitReturns": true, 14 | "noImplicitThis": true, 15 | // "noUnusedLocals": true, 16 | // "noUnusedLocals": true, // cannot use because ANTLR generated files have this problem 17 | // "preserveConstEnums": true, 18 | 19 | "experimentalDecorators": true, 20 | "composite": true 21 | }, 22 | "include":[], 23 | 24 | "references": [ 25 | { 26 | "path": "./src" 27 | }, 28 | { 29 | "path": "./tests" 30 | }, 31 | ] 32 | } -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import commonjs from 'rollup-plugin-commonjs'; 2 | import nodeResolve from 'rollup-plugin-node-resolve'; 3 | import builtins from 'rollup-plugin-node-builtins'; 4 | import globals from 'rollup-plugin-node-globals'; 5 | import wasm from 'rollup-plugin-wasm'; 6 | 7 | export function disallowedImports() { 8 | return { 9 | resolveId: module => { 10 | if (module === 'util') { 11 | throw new Error('Cannot import from Node Util.'); 12 | } 13 | return null; 14 | } 15 | }; 16 | } 17 | export default { 18 | input: 'build/src/index.js', 19 | output: { 20 | file: 'build/diel.js', 21 | format: 'iife', 22 | name: 'diel' 23 | }, 24 | plugins: [nodeResolve({browser: true}), commonjs(), globals(), builtins(), wasm()], 25 | external: ['sql.js'], 26 | globals: { 27 | "sql.js": "initSqlJs", 28 | }, 29 | }; -------------------------------------------------------------------------------- /testEndToEnd/diel/student_range_multi.diel: -------------------------------------------------------------------------------- 1 | create event table score_low(val integer); 2 | create event table score_high(val integer); 3 | 4 | create view current_score_range2 as 5 | select score_low.val as low, score_high.val as high 6 | from latest score_low, latest score_high; 7 | 8 | create event view matching_students2 AS 9 | select students.first_name 10 | from students 11 | natural join exam 12 | join current_score_range2 13 | where (exam.score < current_score_range2.high or current_score_range2.high is null) 14 | and (exam.score > current_score_range2.low or current_score_range2.low is null); 15 | 16 | create output current_matching_students2 AS 17 | select first_name 18 | from matching_students2 19 | where request_timestep = ( 20 | select max(timestep) 21 | from ( 22 | select timestep from score_low 23 | union 24 | select timestep from score_high 25 | ) 26 | ); -------------------------------------------------------------------------------- /src/parser/visitorHelper.ts: -------------------------------------------------------------------------------- 1 | import { DielDataType } from "./dielAstTypes"; 2 | import { Interval } from "antlr4ts/misc"; 3 | 4 | export function parseColumnType(str: string) { 5 | switch (str.toLowerCase()) { 6 | case "number": 7 | case "integer": 8 | case "int": 9 | case "real": 10 | return DielDataType.Number; 11 | case "string": 12 | case "text": 13 | return DielDataType.String; 14 | case "boolean": 15 | return DielDataType.Boolean; 16 | case "datetime": 17 | return DielDataType.TimeStamp; 18 | default: 19 | throw new Error(`parseColumnType error, got ${str}`); 20 | } 21 | } 22 | 23 | export function getCtxSourceCode(ctx: any): string { 24 | if (ctx) { 25 | let a = ctx.start.startIndex; 26 | let b = ctx.stop.stopIndex; 27 | let interval = new Interval(a, b); 28 | return ctx.start.inputStream.getText(interval); 29 | } 30 | return ""; 31 | } 32 | -------------------------------------------------------------------------------- /src/compiler/codegen/staticSql.ts: -------------------------------------------------------------------------------- 1 | // this file contains the static parts of DIEL queries 2 | 3 | export const StaticSql = ` 4 | create table __scales ( 5 | component text not null, 6 | dimension integer not null, 7 | outputName text not null, 8 | x text, 9 | y text, 10 | z text 11 | ); 12 | 13 | create table allInputs ( 14 | timestep integer primary key, 15 | -- DEFAULT (STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')) 16 | -- timestamp kept track of in the JS code 17 | timestamp DATETIME not null, 18 | inputRelation text not null, 19 | request_timestep integer 20 | ); 21 | 22 | create table __perf ( 23 | timestep integer, 24 | kind text, 25 | ts integer, 26 | check(kind = 'start' or kind = 'end'), 27 | primary key(timestep, kind) 28 | ); 29 | 30 | create view __perfByStep as 31 | select 32 | b.timestep, round(e.ts - b.ts, 2) as total 33 | from 34 | (select * from __perf where kind = 'begin') b 35 | join (select * from __perf where kind = 'end') e 36 | on b.timestep = e.timestep; 37 | `; -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import DielRuntime from "./runtime/DielRuntime"; 2 | import { WorkerConfig, SocketConfig, DbSetupConfig } from "./runtime/DbEngine"; 3 | import { DbType, RelationObject, RecordObject } from "./runtime/runtimeTypes"; 4 | import { DerivedRelation, RelationSelection, RelationNameType, CompositeSelection, ColumnNameType, OriginalRelation, HasDefault } from "./parser/dielAstTypes"; 5 | import { GetAllStaticOriginalTables } from "./compiler/DielAstGetters"; 6 | import { GenerateStrFromDielDerivedRelation } from "./compiler/codegen/codeGenSql"; 7 | 8 | export { 9 | DielRuntime, 10 | GenerateStrFromDielDerivedRelation, 11 | // the following are types that need to be exposed 12 | CompositeSelection, 13 | RelationSelection, 14 | RelationObject, 15 | RecordObject, 16 | DbSetupConfig, 17 | WorkerConfig, 18 | SocketConfig, 19 | DerivedRelation, 20 | OriginalRelation, 21 | // not sure if the following should be exposed 22 | DbType, 23 | RelationNameType, 24 | ColumnNameType, 25 | // getters 26 | GetAllStaticOriginalTables, 27 | // shared utils 28 | HasDefault, 29 | }; -------------------------------------------------------------------------------- /src/parser/customError.ts: -------------------------------------------------------------------------------- 1 | import { ANTLRErrorListener, RecognitionException, Recognizer } from "antlr4ts"; 2 | import { ReportDielBasicParsingError } from "../util/messages"; 3 | 4 | // modified from the original src/ConsoleErrorListener.ts 5 | export class ConsoleErrorListener implements ANTLRErrorListener { 6 | /** 7 | * Provides a default instance of {@link ConsoleErrorListener}. 8 | */ 9 | public static readonly INSTANCE: ConsoleErrorListener = new ConsoleErrorListener(); 10 | 11 | /** 12 | * {@inheritDoc} 13 | * 14 | *

15 | * This implementation prints messages to {@link System#err} containing the 16 | * values of {@code line}, {@code charPositionInLine}, and {@code msg} using 17 | * the following format.

18 | * 19 | * 
20 | 	 * line line:charPositionInLine msg
21 | 	 *
22 | */ 23 | public syntaxError( 24 | recognizer: Recognizer, 25 | offendingSymbol: T, 26 | line: number, 27 | charPositionInLine: number, 28 | msg: string, 29 | e: RecognitionException | undefined): void { 30 | ReportDielBasicParsingError(`line ${line}:${charPositionInLine} ${msg}`); 31 | } 32 | } -------------------------------------------------------------------------------- /tests/compilerTests/testOperators.ts: -------------------------------------------------------------------------------- 1 | // TODO 2 | const logicalComposition = ` 3 | create view vComp as SELECT a, b from t where (a > 2 and b >2) or (a < 1 and b < 1);`; 4 | 5 | const comparison = ` 6 | create view v1 as 7 | select day 8 | from xBrushItx 9 | WHERE chart = 'day' 10 | and day <= high and day >=low;`; 11 | 12 | const groupby = ` 13 | create view t as select day as x, count(*) as y 14 | from flights 15 | group by day;`; 16 | 17 | const orderby = ` 18 | create view t2 as select day, delay, carrier from flights order by day, carrier ASC;`; 19 | 20 | const combined = ` 21 | create view t3 as select day as x, count(*) as y 22 | from flights 23 | group by day order by x ASC;`; 24 | 25 | const limit = ` 26 | create view t4 as select day from flights limit 1;`; 27 | 28 | const countNoInput = ` 29 | create view t6 as select count() from testTable;`; 30 | 31 | const limitComplex = ` 32 | create view t5 as select day from flights 33 | limit ( 34 | select count() 35 | from testTable 36 | );`; 37 | 38 | const countStar = ` 39 | create view t7 as select count(*) from testTable;`; 40 | 41 | const countSpecific = ` 42 | create view t7 as select count(aColumn) from testTable;`; -------------------------------------------------------------------------------- /docs/evaluation.md: -------------------------------------------------------------------------------- 1 | # Evaluation of the DIEL framework 2 | 3 | Three approaches: 4 | * run the numberts 5 | * tell Richard's story 6 | * Github sweep 7 | 8 | 9 | We first evaluate the overhead added by using DIEL, c. 10 | [ ] get wasm working with webpack & webworkers 11 | [ ] evaluate the overhead of DIEL with very simple/small datasets vs running in arrays in vanilla JS 12 | [ ] make the previous small dataset larger and compare against parallelism with webworkers 13 | [ ] compare webworker parallelization perf with crossfilter.js and see where crossfitler breaks 14 | [ ] show numbers for connecting to local postgres instance 15 | 16 | The number of resulting Github repos is not that large. 17 | 18 | My hypothesis is that people do not publish it unless its a generalizable tool --- I suspect tha tthe market is probably people using Tableau/Looker, or private adhoc tools. Github has a bias for tool-based repos. 19 | 20 | Maybe our better bet is looking at notebooks --- where people already write SQL(or pandas) queries and generate visualizations. 21 | 22 | ## WebWorker Demo 23 | 24 | chart 1 -- filter on t1 25 | 26 | chart 2 ---- same filter on t2 27 | 28 | 29 | table t1 (load t1 into worker) 30 | 31 | table t2 (load t2 into worker) 32 | -------------------------------------------------------------------------------- /src/parser/sqlAstTypes.ts: -------------------------------------------------------------------------------- 1 | import { Column, RelationConstraints, CompositeSelectionUnit, Command, CompositeSelection, HasDefault } from "./dielAstTypes"; 2 | 3 | export const enum SqlRelationType { 4 | View = "View", 5 | Table = "Table", 6 | } 7 | 8 | interface SqlRelationBase { 9 | rName: string; 10 | relationType: SqlRelationType; 11 | isDynamic?: HasDefault; 12 | } 13 | 14 | export interface SqlOriginalRelation extends SqlRelationBase { 15 | columns: Column[]; 16 | constraints?: RelationConstraints; 17 | } 18 | 19 | export interface SqlDerivedRelation extends SqlRelationBase { 20 | selection: CompositeSelection; 21 | } 22 | 23 | export type SqlRelation = SqlDerivedRelation | SqlOriginalRelation; 24 | 25 | export interface TriggerAst { 26 | tName: string; 27 | afterRelationName: string; 28 | commands: Command[]; 29 | } 30 | 31 | /** 32 | * Note 33 | * - Recycling the programsIr from DIEL AST 34 | * - Note that the name for programsIr in triggers will not be the original relation but a hashed value 35 | */ 36 | export interface SqlAst { 37 | relations: SqlRelation[]; 38 | triggers: TriggerAst[]; 39 | commands: Command[]; 40 | } 41 | 42 | export function CreateEmptySqlAst(): SqlAst { 43 | return { 44 | relations: [], 45 | commands: [], 46 | triggers: [], 47 | }; 48 | } -------------------------------------------------------------------------------- /testEndToEnd/diel/cache.diel: -------------------------------------------------------------------------------- 1 | -- remote table data(datum integer) 2 | 3 | create event table click (num integer); 4 | create event table slider (position integer); 5 | 6 | create view pos_view as select max(position) as position from slider; 7 | 8 | 9 | create event view myclicks as select num*num as squared, subqry.position as pos 10 | from data join click on (datum = num) 11 | join pos_view as subqry; 12 | 13 | create output o1 as select distinct squared, pos from myclicks; 14 | 15 | 16 | -- create event view myclicks as select num * num as squared, position as pos 17 | -- from data2 join click on (datum = num) 18 | -- join latest slider; 19 | 20 | -- Doesn't work. 21 | -- create event view myclicks as select num * num as squared, subqry.position as pos 22 | -- from data join click on (datum = num) 23 | -- join (select max(position) as position from slider) as subqry; 24 | 25 | 26 | 27 | ------------------------------------- 28 | 29 | 30 | -- below doesn't work for some reason (though it should work now?) 31 | -- create output o1 as select subqry.result from (select max(squared) as result from myclicks) as subqry; 32 | 33 | -- create output o1 as select max(squared) as result from myclicks; 34 | -- create event view myclicks as select num * num as squared, max_pos from data join click on datum = num join (select max(position) as max_pos from slider) as subquery; 35 | -------------------------------------------------------------------------------- /tests/parserTests/functionTest.ts: -------------------------------------------------------------------------------- 1 | import { ExprFunAst, FunctionType, DerivedRelation, DielAst, ExprType } from "../../src/parser/dielAstTypes"; 2 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 3 | import { GetRelationDef } from "../../src/compiler/DielAstGetters"; 4 | 5 | export function assertFunctionParsing(ast: DielAst, q: string) { 6 | const logger = GenerateUnitTestErrorLogger("assertFunctionParsing", q); 7 | const v3Relation = GetRelationDef(ast, "v3") as DerivedRelation; 8 | const v3Where = (v3Relation.selection.compositeSelections[0].relation.whereClause as ExprFunAst); 9 | if (v3Where.functionType !== FunctionType.BuiltIn) { 10 | logger.error(`Didn't parse the function properly, Got: ${JSON.stringify(v3Where, null, 2)}`); 11 | } 12 | const v4Relation = GetRelationDef(ast, "v4") as DerivedRelation; 13 | const v4SelectClause = v4Relation.selection.compositeSelections[0].relation.derivedColumnSelections[0]; 14 | if (v4SelectClause.expr.exprType !== ExprType.Func) { 15 | logger.error(`Didn't parse the function for v4 properly---we expected the datetime function, but got: ${JSON.stringify(v4SelectClause, null, 2)}`); 16 | } 17 | const funcExpr = v4SelectClause.expr as ExprFunAst; 18 | if (funcExpr.args.length !== 2) { 19 | logger.error(`Datetime was called with column a and string!, but got ${JSON.stringify(funcExpr.args, null, 2)}`); 20 | } 21 | logger.pass(); 22 | return true; 23 | } -------------------------------------------------------------------------------- /tests/compilerTests/testMaterializationRuntime.ts: -------------------------------------------------------------------------------- 1 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 2 | import { ParsePlainDielAst, CompileAst } from "../../src/compiler/compiler"; 3 | import { DbType } from "../../src"; 4 | import { LogicalTimestep, Relation } from "../../src/parser/dielAstTypes"; 5 | import { DielPhysicalExecution, LocalDbId } from "../../src/compiler/DielPhysicalExecution"; 6 | 7 | export function testMaterializationRuntime() { 8 | // TODO 9 | const query = ` 10 | create event table t1 (a integer); 11 | create event table t2 (a integer); 12 | create event table t3 (a integer); 13 | 14 | create view v1 as select a + 1 as aPrime from t1 where a > 2; 15 | 16 | create output o1 as select aPrime from v1 join t2 on aPrime = a; 17 | create output o2 as select aPrime from v1 join t3 on aPrime = a; 18 | `; 19 | 20 | const logger = GenerateUnitTestErrorLogger("testMaterializationRuntime", query); 21 | const ast = ParsePlainDielAst(query); 22 | CompileAst(ast); 23 | 24 | const physicalMetaData = { 25 | dbs: new Map([[1, {dbType: DbType.Local}]]), 26 | relationLocation: new Map() 27 | }; 28 | const getEventByTimestep = (n: LogicalTimestep) => ""; 29 | 30 | const addRelationToDielMock = (r: Relation) => {}; 31 | const physicalExecution = new DielPhysicalExecution(ast, physicalMetaData, getEventByTimestep, addRelationToDielMock); 32 | const sqlAst = physicalExecution.getAstFromDbId(LocalDbId); 33 | // assertions on sqlAst for materialization 34 | } -------------------------------------------------------------------------------- /testEndToEnd/diel/simple.diel: -------------------------------------------------------------------------------- 1 | create event table click(num integer); 2 | 3 | create event table slider (position integer); 4 | 5 | create event view myclicks as 6 | select num * num as squared, subqry.position as pos 7 | from data join click on (datum = num) 8 | join (select max(position) as position from slider) as subqry; 9 | 10 | -- select origin, count() from flights group by origin limit 20; 11 | 12 | -- remote 13 | 14 | -- create table data(datum integer); 15 | 16 | -- create event table click (num integer); 17 | -- create event table slider (position integer); 18 | 19 | -- create event view pos_view as select position as pos from slider; 20 | 21 | -- create event view myclicks as select num * num as squared, pos 22 | -- from data join click on (datum = num) join pos_view; 23 | 24 | -- create output o1 as select squared, pos from myclicks; 25 | 26 | -- create event view myclicks as select num * num as squared from 27 | -- data join click on datum = num; 28 | 29 | -- -- create output o1 as select max(squared) from myclicks; 30 | 31 | -- -- below doesn't work for some reason 32 | 33 | -- create output o1 as select subqry.result from (select max(squared) as result from myclicks) as subqry; 34 | 35 | -- create output allOriginAirports AS 36 | -- select * from allOriginAirportsEvent; 37 | 38 | -- create event view allOriginAirportsEvent AS 39 | -- select 40 | -- origin, 41 | -- count() as c 42 | -- from flights 43 | -- group by origin 44 | -- order by c 45 | -- limit 10; 46 | -------------------------------------------------------------------------------- /tests/compilerTests/assertDecomposition.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This function generates two different queries 3 | * describing the distribution based on column types 4 | * 5 | * the results should be generated as ASTs and mapped to strings 6 | * via the functions in codeGenSql.ts 7 | */ 8 | export function assertSingleColumnNaive() { 9 | let q = ` 10 | create input t1 ( 11 | a text, 12 | b int, 13 | ); 14 | create view scatterT1 as select a, b from t1; 15 | `; 16 | // expect: array 17 | const expectedResult = [ 18 | `select count(), a from t1 group by a;`, 19 | `select count(), b from t1 group by b;` 20 | ]; 21 | } 22 | 23 | /** 24 | * much harder 25 | * Need to run basic statistic on the table 26 | * - store that in some metadata construct and generate ASTs based on that 27 | */ 28 | export function assertSingleColumnUniqueValuesAware() { 29 | let q = ` 30 | create input tMany ( 31 | a text, 32 | b int, 33 | ); 34 | create input tFew ( 35 | a text, 36 | b int, 37 | ); 38 | -- TODO 39 | insert into tMany (a, b) values (); 40 | insert into tFew (a, b) values ('hello', 1), ('world', 2); 41 | create view scatterT1 as select a, b from t1; 42 | `; 43 | // expect: array 44 | // FIXME: below is not name adapted 45 | const expectedTManyResult = [ 46 | `create view t1BucketByB AS 47 | select INT(b / 100) * 100 as bBucketCoarse, INT(b / 10) * 10 from t1; 48 | select count(), bBucket from t1BucketByB group by bBucket; 49 | `, 50 | `select count(), b from t1 group by b;` 51 | ]; 52 | } -------------------------------------------------------------------------------- /tests/compilerTests/assertNormalization.ts: -------------------------------------------------------------------------------- 1 | import { DerivedRelation, ExprColumnAst, DielAst } from "../../src/parser/dielAstTypes"; 2 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 3 | import { GetRelationDef } from "../../src/compiler/DielAstGetters"; 4 | 5 | export function assertBasicNormalizationOfRelation(ast: DielAst, q: string) { 6 | const logger = GenerateUnitTestErrorLogger("assertBasicNormalizationOfRelation", q); 7 | const v1Relation = GetRelationDef(ast, "v1") as DerivedRelation; 8 | const aSelection = v1Relation.selection.compositeSelections[0].relation.derivedColumnSelections[0].expr as ExprColumnAst; 9 | if (aSelection.relationName !== "t1") { 10 | logger.error(`Normalization pass failed, I had expected a to be matched with relation t1. Got: ${JSON.stringify(aSelection, null, 2)}`); 11 | } 12 | 13 | // the following are known bugs, currently not implemented. 14 | 15 | // const v2Relation = ir.allDerivedRelations.get("v2"); 16 | // // make sure that the subquery is at least properly derived 17 | // const v2Subquery = v2Relation[0].relation.joinClauses[0].relation.subquery.compositeSelections[0].relation; 18 | // if (!v2Subquery.derivedColumnSelections) { 19 | // logger(`Normalization pass failed to normalize subquery, didn't perform the act. Got: ${JSON.stringify(v2Subquery, null, 2)}`); 20 | // } 21 | 22 | // const v2bSelection = v2Subquery.derivedColumnSelections[0].expr as ExprColumnAst; 23 | // if (v2bSelection.relationName !== "t2") { 24 | // logger(`Normalization pass failed to find t2 for subquery`); 25 | // } 26 | return true; 27 | } -------------------------------------------------------------------------------- /testEndToEnd/cacheTest.ts: -------------------------------------------------------------------------------- 1 | import * as path from "path"; 2 | import { DielRuntime, DbSetupConfig, DbType, RelationObject } from "../src"; 3 | import { LogInternalError, DielInternalErrorType } from "../src/util/messages"; 4 | 5 | const jsFile = path.resolve(__dirname, "../../..//node_modules/sql.js/dist/worker.sql.js"); 6 | 7 | const dbConfigs: DbSetupConfig[] = [{ 8 | dbType: DbType.Worker, 9 | jsFile, 10 | dataFile: path.resolve(__dirname, "../../testEndToEnd/data/cache.sqlite") 11 | }, 12 | ]; 13 | 14 | const mainDbPath: string = null; 15 | const dielFiles = [path.resolve(__dirname, "../../testEndToEnd/diel/cache.diel")]; 16 | const isCaching = true; 17 | 18 | const answerByRequestTimestep = new Map(); 19 | // timesteps start on 2 since the first 1 is used at setup time 20 | answerByRequestTimestep.set(2, 50); 21 | answerByRequestTimestep.set(3, 85); 22 | answerByRequestTimestep.set(4, 50); 23 | 24 | export function testSimpleCache() { 25 | 26 | const diel = new DielRuntime({ 27 | isStrict: true, 28 | showLog: true, 29 | setupCb: testCache, 30 | caching: isCaching, 31 | dielFiles, 32 | mainDbPath, 33 | dbConfigs, 34 | }); 35 | 36 | async function testCache() { 37 | console.log(`Cache test with${isCaching ? "" : " no"} caching starting`); 38 | 39 | diel.BindOutput("o1", (o: RelationObject) => { 40 | console.log("results!", o); 41 | }); 42 | 43 | diel.NewInput("click", {num: 10}); 44 | diel.NewInput("slider", {position: 105}); 45 | diel.NewInput("slider", {position: 103}); 46 | diel.NewInput("slider", {position: 109}); 47 | 48 | /* 49 | const rName = await diel.AddOutputRelationByString(` 50 | select datum from data; 51 | `); 52 | */ 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /testEndToEnd/sensorTest.ts: -------------------------------------------------------------------------------- 1 | import * as path from "path"; 2 | import { DielRuntime, DbSetupConfig, DbType, RelationObject } from "../src"; 3 | import { LogInternalError, LogTest, LogInternalWarning } from "../src/util/messages"; 4 | 5 | const jsFile = path.resolve(__dirname, "../../..//node_modules/sql.js/dist/worker.sql.js"); 6 | 7 | const dbConfigs: DbSetupConfig[] = [{ 8 | dbType: DbType.Worker, 9 | jsFile, 10 | dataFile: path.resolve(__dirname, "../../testEndToEnd/data/sensors_10000.sqlite") 11 | }, 12 | ]; 13 | 14 | const mainDbPath: string = null; 15 | const dielFiles = [path.resolve(__dirname, "../../testEndToEnd/diel/sensors.diel")]; 16 | 17 | export function sensorTest(perf: (diel: DielRuntime) => void) { 18 | 19 | const diel = new DielRuntime({ 20 | isStrict: true, 21 | showLog: true, 22 | setupCb: testClass, 23 | caching: false, 24 | dielFiles, 25 | mainDbPath, 26 | dbConfigs, 27 | }); 28 | 29 | async function testClass() { 30 | // diel.BindOutput("current_time_selection_pretty", (o: RelationObject) => { 31 | // console.log("current_time_selection_pretty", o); 32 | // }); 33 | diel.BindOutput("pack_break_regen", (o: RelationObject) => { 34 | console.log("pack_break_regen", o); 35 | }); 36 | // diel.BindOutput("pack_ther", (o: RelationObject) => { 37 | // console.log("pack_ther", o); 38 | // }); 39 | // diel.BindOutput("pack_cell", (o: RelationObject) => { 40 | // console.log("pack_cell", o); 41 | // }); 42 | diel.NewInput("time_selection", {minTs: null, maxTs: null}); 43 | diel.NewInput("time_selection", {minTs: 2458433.3161339, maxTs: 2458433.36517046}); 44 | // window.setTimeout(() => { 45 | // perf(diel); 46 | // }, 5000); 47 | } 48 | return diel; 49 | } 50 | -------------------------------------------------------------------------------- /tests/testEventTableCacheCreation.ts: -------------------------------------------------------------------------------- 1 | import { DbType, DerivedRelation } from "../src"; 2 | import { DielPhysicalExecution } from "../src/compiler/DielPhysicalExecution"; 3 | import { CompileAst, ParsePlainDielAst } from "../src/compiler/compiler"; 4 | import { LogicalTimestep, Relation } from "../src/parser/dielAstTypes"; 5 | import { generateStringFromSqlIr, GenerateSqlRelationString } from "../src/compiler/codegen/codeGenSql"; 6 | import { GetCachedEventView } from "..//src/compiler/passes/distributeQueries"; 7 | import { GetRelationDef } from "..//src/compiler/DielAstGetters"; 8 | 9 | export function testTriTableCreation() { 10 | const query = ` 11 | create table data (datum integer); 12 | create event table click (num integer, pos integer); 13 | create event view myclicks as select num * num as squared, pos from click 14 | join data on datum = num; 15 | 16 | create output o1 as select max(squared) as maximum from myclicks; 17 | `; 18 | const physicalMetaData = { 19 | dbs: new Map([[1, {dbType: DbType.Local}], [2, {dbType: DbType.Worker}]]), 20 | relationLocation: new Map(), 21 | cache: true, 22 | }; 23 | 24 | physicalMetaData.relationLocation.set("data", {dbType: 2}); 25 | 26 | const ast = ParsePlainDielAst(query); 27 | CompileAst(ast); 28 | const getEventByTimestep = (n: LogicalTimestep) => ""; 29 | 30 | const addRelationToDielMock = (r: Relation) => {}; 31 | let physicalExecution = new DielPhysicalExecution(ast, physicalMetaData, getEventByTimestep, addRelationToDielMock); 32 | 33 | let sqlAst = physicalExecution.sqlAstSpecPerDb.get(1); 34 | 35 | let cacheTriplet = GetCachedEventView(GetRelationDef(ast, "myclicks") as DerivedRelation, true); 36 | 37 | console.log(GenerateSqlRelationString(cacheTriplet.cacheTable)); 38 | 39 | 40 | } 41 | -------------------------------------------------------------------------------- /tests/testHelper.ts: -------------------------------------------------------------------------------- 1 | import { ExprAst, ExprType, ExprFunAst, ExprColumnAst, ExprValAst } from "../src/parser/dielAstTypes"; 2 | import { TestLogger } from "./testTypes"; 3 | import { BgYellow, Reset, FgRed, FgGreen } from "../src/util/messages"; 4 | 5 | // assert expression is a function 6 | export function assertExprAsFunctionWithName(e: ExprAst, fName: string) { 7 | if (e.exprType !== ExprType.Func) { 8 | return false; 9 | } 10 | if ((e).functionReference !== fName) { 11 | return false; 12 | } 13 | return true; 14 | } 15 | 16 | // assert expression is a number 17 | // the any might be a bit brittle 18 | export function assertValue(e: ExprAst, val: any) { 19 | if (e.exprType !== ExprType.Val) { 20 | return false; 21 | } 22 | if ((e).value !== val) { 23 | console.log(`got value ${val} instead`); 24 | return false; 25 | } 26 | return true; 27 | } 28 | 29 | // assert expression is a sepecific column and return arguments 30 | export function assertExprAsColumnWithname(e: ExprAst, cName: string) { 31 | if (e.exprType !== ExprType.Column) { 32 | return false; 33 | } 34 | if ((e).columnName !== cName) { 35 | return false; 36 | } 37 | return true; 38 | } 39 | 40 | export function GenerateUnitTestErrorLogger(testName: string, q?: string): TestLogger { 41 | console.log(`=================================\n 42 | ${BgYellow}Starting Test: %s${Reset}\n`, testName); 43 | if (q) console.log(`With query:\n%s`, q); 44 | return { 45 | error: (m: string, objs: any | any[]) => { 46 | console.log(`\n ${FgRed}Error for [${testName}]: %s${Reset}`, m, objs); 47 | throw new Error(`Test [${testName}] failed\n`); 48 | }, 49 | pass: () => { 50 | console.log(`${FgGreen}Test [${testName}] Passed :) \n================================${Reset}`); 51 | } 52 | }; 53 | } -------------------------------------------------------------------------------- /docs/github.md: -------------------------------------------------------------------------------- 1 | # Survey of Projects on Github 2 | 3 | [Searching for language with both sql and js](https://github.com/search?q=language%3A+sql+language%3Ajs) 4 | 5 | Wow [adding topic of visualization](https://github.com/search?q=language%3Asql+topic%3Avisualization) yielded none --- part of that is sometimes people just don't label visualization, and part of it even when people are generating SQL, there is no lable (since I think it' just based on the extention of the file name). 6 | 7 | Whereas [just JS and vis](https://github.com/search?q=language%3Ajs+topic%3Avisualization) gave 1K. 8 | 9 | (sql with js and visualization)[https://github.com/search?q=sql+language%3Ajs+topic%3Avisualization] gave 5 results, again lossy, since Franchise didn't even show up. 10 | 11 | I think the issue is that these are mostly libraries. 12 | 13 | 14 | My hypothesis is that people do not publish it unless its a generalizable tool --- I suspect tha tthe market is probably people using Tableau/Looker, or private adhoc tools. Github has a bias for tool-based repos. 15 | 16 | Maybe our better bet is looking at notebooks --- where people already write SQL(or pandas) queries and generate visualizations. 17 | 18 | 19 | ## Write SQL* and Chart the Results 20 | \*by sql we include related, such as dataframes 21 | 22 | These tools show up after searching [sql & visualization](https://github.com/search?q=sql+visualization). 23 | 24 | 25 | [Plotly, Free, open-source SQL client for Windows and Mac](https://plot.ly/free-sql-client-download/) (>700 stars) 26 | [Looker OS clone](https://github.com/mariusandra/insights) (>500 stars) 27 | [lumify](https://github.com/lumifyio/lumify/) (>300 stars) 28 | 29 | Tellingly, [big data & visualization](https://github.com/search?q=sql+visualization) 30 | 31 | ## Custom Charts 32 | 33 | [crossfilter](https://github.com/omnisci/mapd-crossfilter) (>30 stars) 34 | 35 | 36 | ## API docs 37 | [Search methods]( 38 | https://help.github.com/en/articles/searching-for-repositories#search-by-language) -------------------------------------------------------------------------------- /tests/compilerTests/assertTypes.ts: -------------------------------------------------------------------------------- 1 | import { ParsePlainDielAst, CompileAst } from "../../src/compiler/compiler"; 2 | import { DielDataType } from "../../src/parser/dielAstTypes"; 3 | import { getColumnTypeFromRelation } from "../../src/compiler/passes/inferType"; 4 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 5 | import { GetRelationDef } from "../../src/compiler/DielAstGetters"; 6 | 7 | export function assertMultiplyType() { 8 | const q = ` 9 | create event table t (a int); 10 | create view v2 as select a*2 as newA from t; 11 | `; 12 | const logger = GenerateUnitTestErrorLogger("assertMultiplyType", q); 13 | let ast = ParsePlainDielAst(q); 14 | CompileAst(ast); 15 | const v2 = GetRelationDef(ast, "v2"); 16 | const v2Type = getColumnTypeFromRelation(v2, "newA"); 17 | if (v2Type !== DielDataType.Number) { 18 | logger.error(`Column "newA" of "v2" is not correctly typed, got ${v2Type} instead`); 19 | } 20 | logger.pass(); 21 | return true; 22 | } 23 | 24 | export function assertSimpleType() { 25 | 26 | let q = ` 27 | create event table Attendance ( 28 | arrival int, 29 | departure int, 30 | aid int 31 | ); 32 | 33 | create event table Attendee ( 34 | aid int primary key, 35 | area text 36 | ); 37 | create view v1Prime as select a.arrival from Attendance a; 38 | create view v1 as select arrival from Attendance; 39 | `; 40 | const logger = GenerateUnitTestErrorLogger("assertSimpleType", q); 41 | let ast = ParsePlainDielAst(q); 42 | // then we need to compile it! 43 | CompileAst(ast); 44 | function arrivalAssert(viewName: string) { 45 | const viewAst = GetRelationDef(ast, viewName); 46 | const arrivalType = getColumnTypeFromRelation(viewAst, "arrival"); 47 | if (arrivalType !== DielDataType.Number) { 48 | logger.error(`Column "arrival" of ${viewName} is not correctly typed, got ${arrivalType} instead`); 49 | } 50 | } 51 | arrivalAssert("v1Prime"); 52 | arrivalAssert("v1"); 53 | logger.pass(); 54 | return true; 55 | } 56 | 57 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "diel", 3 | "version": "1.0.9", 4 | "description": "data interaction events log is a new SQL embeded DSL for managing the state of history-aware interactive applications", 5 | "main": "build/diel.js", 6 | "jsdelivr": "build/diel.js", 7 | "module": "build/src/index.js", 8 | "types": "build/src/index.d.ts", 9 | "files": [ 10 | "build/src/*" 11 | ], 12 | "scripts": { 13 | "setup": "npm install", 14 | "build": "tsc -b src", 15 | "bundle": "rollup -c", 16 | "lang": "antlr4ts -visitor -no-listener src/parser/grammar/DIEL.g4", 17 | "test": "tsc -b tests/ && node build/tests/index.js", 18 | "test-watch": "tsc -b -w tests/", 19 | "debug": "node --inspect-brk -r ts-node/register tests/index.ts", 20 | "getAst": "ts-node tests/getAst.ts", 21 | "testEndToEnd": "webpack-dev-server --mode development --hot" 22 | }, 23 | "repository": { 24 | "type": "git", 25 | "url": "git+https://github.com/yifanwu/diel.git" 26 | }, 27 | "keywords": [ 28 | "interaction", 29 | "history", 30 | "sql" 31 | ], 32 | "author": "yifan wu", 33 | "license": "ISC", 34 | "bugs": { 35 | "url": "https://github.com/yifanwu/diel/issues" 36 | }, 37 | "homepage": "https://github.com/yifanwu/diel#readme", 38 | "devDependencies": { 39 | "antlr4ts-cli": "^0.4.0-alpha.4", 40 | "awesome-typescript-loader": "^5.2.1", 41 | "ignore-loader": "^0.1.2", 42 | "rollup": "^1.9.0", 43 | "rollup-plugin-commonjs": "^9.3.4", 44 | "rollup-plugin-node-builtins": "^2.1.2", 45 | "rollup-plugin-node-globals": "^1.4.0", 46 | "rollup-plugin-node-resolve": "^4.2.4", 47 | "rollup-plugin-wasm": "^3.0.0", 48 | "source-map-loader": "^0.2.3", 49 | "ts-node": "^7.0.1", 50 | "tslint": "^5.15.0", 51 | "tslint-loader": "^3.5.3", 52 | "typescript": "^3.4.1", 53 | "typescript-formatter": "^7.2.2", 54 | "webpack": "^4.34.0", 55 | "webpack-cli": "^3.3.0", 56 | "webpack-dev-server": "^3.3.1" 57 | }, 58 | "dependencies": { 59 | "@types/sql.js": "^1.0.0", 60 | "antlr4ts": "^0.4.1-alpha.0", 61 | "sql.js": "1.0.0" 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | path = require("path") 2 | 3 | module.exports = { 4 | node: { 5 | fs: 'empty' 6 | }, 7 | 8 | 9 | entry: "./testEndToEnd/index.ts", 10 | output: { 11 | filename: "bundle.js", 12 | path: __dirname + "/dist", 13 | publicPath: "/dist", 14 | }, 15 | // Enable sourcemaps for debugging webpack's output. 16 | devtool: "source-map", 17 | 18 | resolve: { 19 | // Add '.ts' and '.tsx' as resolvable extensions. 20 | extensions: [".webpack.js", ".web.js", ".ts", ".js"] 21 | }, 22 | // optimization: { 23 | // minimize: false, 24 | // }, 25 | module: { 26 | noParse: /sql-wasm.wasm/, 27 | rules: [ 28 | // All files with a '.ts' or '.tsx' extension will be handled by 'awesome-typescript-loader'. 29 | // 30 | { 31 | test: /\.wasm$/, 32 | type: "javascript/auto", 33 | loader: "ignore-loader", 34 | }, 35 | { 36 | test: /\.tsx?$/, 37 | loaders: [ 38 | 'awesome-typescript-loader?{configFileName: "testEndToEnd/tsconfig.json"}', 39 | ], 40 | } 41 | // { 42 | // test: /\.js$/, 43 | // enforce: "pre", 44 | // loader: "source-map-loader", 45 | // exclude: "/node_modules/" 46 | // }, 47 | // { 48 | // test: /\.tsx?$/, 49 | // enforce: "pre", 50 | // loader: 'tslint-loader' 51 | // } 52 | ], 53 | }, 54 | 55 | // When importing a module whose path matches one of the following, just 56 | // assume a corresponding global variable exists and use that instead. 57 | // This is important because it allows us to avoid bundling all of our 58 | // dependencies, which allows browsers to cache those libraries between builds. 59 | externals: { 60 | // "sql.js": "SQL", 61 | //"sql.js": "initSqlJs", 62 | }, 63 | 64 | target: 'web', 65 | devServer: { 66 | // publicPath: "/dist", 67 | inline: true 68 | }, 69 | }; 70 | -------------------------------------------------------------------------------- /tests/sqlCodeGenTest.ts: -------------------------------------------------------------------------------- 1 | import { SqlStrFromSelectionUnit } from "../src/compiler/codegen/codeGenSql"; 2 | import { ExprType, FunctionType, DielDataType, SelectionUnit, JoinType, AstType, RelationReferenceType } from "../src/parser/dielAstTypes"; 3 | import { GenerateUnitTestErrorLogger } from "./testHelper"; 4 | 5 | /** 6 | * stripped of space and lower cased 7 | */ 8 | function strip(s: string) { 9 | return s.replace(/\s|\n+/g, "").toLowerCase(); 10 | } 11 | 12 | export function codeGenBasicSQLTest() { 13 | // take an ast and assert the output 14 | const columns = [{ 15 | expr: { 16 | exprType: ExprType.Column, 17 | columnName: "aId", 18 | relationName: "t1" 19 | }, 20 | alias: "a" 21 | }, 22 | { 23 | expr: { 24 | exprType: ExprType.Star, 25 | relationName: "t2" 26 | } 27 | } 28 | ]; 29 | const predicate = { 30 | exprType: ExprType.Func, 31 | dataType: DielDataType.Boolean, 32 | functionType: FunctionType.Compare, 33 | functionReference: "=", 34 | args: [ 35 | { 36 | exprType: ExprType.Column, 37 | columnName: "aId", 38 | relationName: "t1" 39 | }, 40 | { 41 | exprType: ExprType.Column, 42 | columnName: "aId", 43 | relationName: "t2" 44 | } 45 | ] 46 | }; 47 | const ast: SelectionUnit = { 48 | derivedColumnSelections: columns, 49 | columnSelections: columns, 50 | baseRelation: { 51 | relationReferenceType: RelationReferenceType.Direct, 52 | relationName: "t1" 53 | }, 54 | joinClauses: [{ 55 | astType: AstType.Join, 56 | joinType: JoinType.LeftOuter, 57 | relation: { 58 | relationReferenceType: RelationReferenceType.Direct, 59 | relationName: "t2" 60 | }, 61 | predicate 62 | }] 63 | }; 64 | const sGen = SqlStrFromSelectionUnit(ast); 65 | const refSolution = `select t1.aId as a, t2.* from t1 LEFT OUTER join t2 on t1.aId = t2.aId`; 66 | const logger = GenerateUnitTestErrorLogger("codeGenBasicSQLTest", refSolution); 67 | if (strip(sGen) !== strip(refSolution)) { 68 | logger.error(`Generated SQL is wrong. Expected ${refSolution}, but got ${sGen} instead`); 69 | } 70 | } -------------------------------------------------------------------------------- /src/compiler/passes/passesHelper.ts: -------------------------------------------------------------------------------- 1 | import { LogInternalError, DielInternalErrorType } from "../../util/messages"; 2 | import { DependencyTree } from "../../runtime/runtimeTypes"; 3 | 4 | export function getTopologicalOrder(depTree: DependencyTree) { 5 | // lots of redundancy for access 6 | // this code is so dumb 7 | let visitedStringToNumber = new Map(); 8 | let visitedArray: { visited: boolean, relationName: string }[] = []; 9 | let topoSorted: string[] = []; 10 | let i = 0; 11 | for (let key of depTree.keys()) { 12 | visitedStringToNumber.set(key, i); 13 | i += 1; 14 | visitedArray.push({visited: false, relationName: key}); 15 | } 16 | let hasUnmarked = visitedArray.filter(v => !v.visited); 17 | let loopCount = 1; 18 | while (hasUnmarked.length > 0) { 19 | topoVisit(hasUnmarked[0].relationName); 20 | hasUnmarked = visitedArray.filter(v => !v.visited); 21 | loopCount += 1; 22 | if (loopCount > 1000) { // this is brittle, and temporarily for debugging #FIXME 23 | LogInternalError(`Too many loops in toplogical sort`); 24 | } 25 | } 26 | function topoVisit(relation: string) { 27 | loopCount += 1; 28 | if (loopCount > 1000) { // this is brittle, and temporarily for debugging #FIXME 29 | debugger; 30 | LogInternalError(`Too many loops in toplogical sort`); 31 | } 32 | if (!visitedStringToNumber.has(relation)) { 33 | // this should be the case where a static relation is referred 34 | // in which case we can just skip; enhance later #FIXMELATER 35 | return; 36 | } 37 | const idx = visitedStringToNumber.get(relation); 38 | if ((idx > -1) && visitedArray[idx].visited) { 39 | return; 40 | } 41 | // ugh of vs in 42 | const node = depTree.get(relation); 43 | if (node) { 44 | for (let d of node.dependsOn) { 45 | topoVisit(d); 46 | } 47 | } 48 | if (idx > -1) { 49 | if (topoSorted.find(t => t === relation)) { 50 | LogInternalError(`Shouldn't be added again, ${relation}`); 51 | } 52 | visitedArray[idx].visited = true; 53 | topoSorted.push(relation); 54 | } 55 | } 56 | // there are no dangling leaves; they will just have no dependencies 57 | return topoSorted; 58 | } -------------------------------------------------------------------------------- /src/compiler/passes/normalizeConstraints.ts: -------------------------------------------------------------------------------- 1 | import { LogInternalError } from "../../util/messages"; 2 | import { OriginalRelation, DielAst } from "../../parser/dielAstTypes"; 3 | import { GetAllDielDefinedOriginalRelations } from "../DielAstGetters"; 4 | 5 | export function NormalizeConstraintsForSingleOriginalRelation(r: OriginalRelation) { 6 | r.columns.map(c => { 7 | if (c.constraints) { 8 | if (c.constraints.notNull) { 9 | if (r.constraints.notNull) { 10 | r.constraints.notNull.push(c.cName); 11 | } else { 12 | r.constraints.notNull = [c.cName]; 13 | } 14 | } 15 | if (c.constraints.unique) { 16 | if (r.constraints.uniques) { 17 | r.constraints.uniques.push([c.cName]); 18 | } else { 19 | r.constraints.uniques = [[c.cName]]; 20 | } 21 | } 22 | if (c.constraints.primaryKey) { 23 | if (r.constraints.primaryKey && (r.constraints.primaryKey.length > 0)) { 24 | LogInternalError(`Cannot have more than one primary key! You already have ${r.constraints.primaryKey} but we are adding ${c.cName}`); 25 | } else { 26 | r.constraints.primaryKey = [c.cName]; 27 | } 28 | } 29 | // not including autoincrement here since it's not really a constraint... sigh semantics 30 | } 31 | }); 32 | } 33 | 34 | /** 35 | * constraints can be created either on the column or on the table 36 | * we will move all to the table level for easier checking 37 | * but i think SQL syntax requires some to be written in column leve, e.g., not null 38 | * however primary key can be in both positions (presumably because it can take multiple columns) 39 | * 40 | * the function will walk through the original tables and move the column level 41 | * constraints to relation constraints 42 | * and maybe label the field as "derived" 43 | * 44 | * see https://www.sqlite.org/syntax/column-constraint.html 45 | * and https://www.sqlite.org/syntax/table-constraint.html 46 | * 47 | * we have augmented it so that it could work with views as well 48 | */ 49 | export function NormalizeConstraints(ast: DielAst) { 50 | GetAllDielDefinedOriginalRelations(ast).map((r) => { 51 | NormalizeConstraintsForSingleOriginalRelation(r); 52 | }); 53 | } 54 | -------------------------------------------------------------------------------- /testEndToEnd/testComplexCaching.ts: -------------------------------------------------------------------------------- 1 | import * as path from "path"; 2 | import { DielRuntime, DbSetupConfig, DbType, RelationObject } from "../src"; 3 | import { LogTest } from "../src/util/messages"; 4 | const jsFile = path.resolve(__dirname, "../../..//node_modules/sql.js/dist/worker.sql.js"); 5 | 6 | 7 | const dbConfigs: DbSetupConfig[] = [{ 8 | dbType: DbType.Worker, 9 | jsFile, 10 | dataFile: path.resolve(__dirname, "../../testEndToEnd/data/students.sqlite") 11 | }, 12 | ]; 13 | 14 | const mainDbPath: string = null; 15 | const isCaching = true; 16 | 17 | export function testRangeCaching() { 18 | const dielFiles = [path.resolve(__dirname, "../../testEndToEnd/diel/students_range.diel")]; 19 | const diel = new DielRuntime({ 20 | isStrict: true, 21 | showLog: true, 22 | setupCb: testClass, 23 | caching: isCaching, 24 | dielFiles, 25 | mainDbPath, 26 | dbConfigs, 27 | }); 28 | 29 | async function testClass() { 30 | console.log(`Cache test with${isCaching ? "" : " no"} caching starting`); 31 | 32 | diel.BindOutput("current_matching_students", (o: RelationObject) => { 33 | LogTest("results!", JSON.stringify(o)); 34 | }); 35 | 36 | diel.NewInput("score_range", {low: 10, high: 80}); 37 | diel.NewInput("score_range", {low: 70, high: 90}); 38 | diel.NewInput("score_range", {low: 70, high: 90}); 39 | diel.NewInput("score_range", {low: 10, high: 80}); 40 | } 41 | } 42 | 43 | 44 | export function testMultiTableCaching() { 45 | const dielFiles = [path.resolve(__dirname, "../../testEndToEnd/diel/student_range_multi.diel")]; 46 | const diel = new DielRuntime({ 47 | isStrict: true, 48 | showLog: true, 49 | setupCb: testClass, 50 | caching: isCaching, 51 | dielFiles, 52 | mainDbPath, 53 | dbConfigs, 54 | }); 55 | 56 | async function testClass() { 57 | console.log(`Cache test with${isCaching ? "" : " no"} caching starting`); 58 | 59 | // diel.BindOutput("current_matching_students2", (o: RelationObject) => { 60 | // LogTest("results!", JSON.stringify(o)); 61 | // }); 62 | 63 | diel.NewInput("score_low", {val: 10}); 64 | diel.NewInput("score_high", {val: 70}); 65 | diel.NewInput("score_low", {val: 20}); 66 | diel.NewInput("score_low", {val: 10}); 67 | } 68 | } -------------------------------------------------------------------------------- /testEndToEnd/diel/sensors.diel: -------------------------------------------------------------------------------- 1 | create event table time_selection ( 2 | minTs integer, 3 | maxTs integer 4 | ); 5 | 6 | create view current_time_selection AS 7 | select * from latest time_selection; 8 | 9 | 10 | -- create output current_time_selection_pretty AS 11 | -- select 12 | -- datetime(minTs, 'unixepoch') as minTs, 13 | -- datetime(maxTs, 'unixepoch') as maxTs 14 | -- from current_time_selection; 15 | 16 | create view current_filtered_vals AS 17 | select * 18 | from log 19 | join current_time_selection s 20 | on (log.ts < s.maxTs 21 | and log.ts > s.minTs) or 22 | (s.maxTs is null); 23 | 24 | -- TODO: this is a good place ot use templates 25 | -- create output pack_ther as 26 | -- select 27 | -- time, 28 | -- ts, 29 | -- min as val, 30 | -- 'minVal' as kind 31 | -- from current_filtered_vals 32 | -- where device = 'pack-ther' 33 | -- union 34 | -- select 35 | -- time, 36 | -- ts, 37 | -- max as val, 38 | -- 'maxVal' as kind 39 | -- from current_filtered_vals 40 | -- where device = 'pack-ther' 41 | -- union 42 | -- select 43 | -- time as time, 44 | -- ts, 45 | -- value as val, 46 | -- 'avgVal' as kind 47 | -- from current_filtered_vals 48 | -- where device = 'pack-ther' 49 | -- ; 50 | 51 | -- create output pack_cell as 52 | -- select 53 | -- time as time, 54 | -- ts, 55 | -- min as val, 56 | -- 'minVal' as kind 57 | -- from current_filtered_vals 58 | -- where device = 'pack-cell' 59 | -- union 60 | -- select 61 | -- time as time, 62 | -- ts, 63 | -- max as val, 64 | -- 'maxVal' as kind 65 | -- from current_filtered_vals 66 | -- where device = 'pack-cell' 67 | -- union 68 | -- select 69 | -- time as time, 70 | -- ts, 71 | -- value as val, 72 | -- 'avgVal' as kind 73 | -- from current_filtered_vals 74 | -- where device = 'pack-cell' 75 | -- ; 76 | 77 | create output pack_break_regen AS 78 | select 79 | time, 80 | ts, 81 | value 82 | from current_filtered_vals 83 | where device = 'petal-brake-regen'; 84 | 85 | 86 | -- // "select time, value from log where device = 'pack-break-regen'", 87 | -- // "select time, value from log where device = 'speed-1'", 88 | -------------------------------------------------------------------------------- /tests/compilerTests/testStarExpantion.ts: -------------------------------------------------------------------------------- 1 | import { LogInfo } from "../../src/util/messages"; 2 | import { ExprColumnAst, DielDataType, DerivedRelation, DielAst } from "../../src/parser/dielAstTypes"; 3 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 4 | import { ParsePlainDielAst, CompileAst } from "../../src/compiler/compiler"; 5 | import { GetRelationDef } from "../../src/compiler/DielAstGetters"; 6 | 7 | export function assertAllStar() { 8 | function assertColumns(viewName: string, selections: {columnName: string, relationName: string, dataType: DielDataType}[]) { 9 | const view = GetRelationDef(ast, viewName) as DerivedRelation; 10 | const columns = view.selection.compositeSelections[0].relation.derivedColumnSelections; 11 | if (!columns) { 12 | logger.error(`${viewName} is not expanded`); 13 | } 14 | selections.forEach((v, idx) => { 15 | const col = columns[idx].expr as ExprColumnAst; 16 | if (col.columnName !== v.columnName) { 17 | logger.error(`${viewName} did not expand properly, expected column name "${v.columnName}" but got ${col.columnName} instead.`); 18 | } 19 | if (col.relationName !== v.relationName) { 20 | logger.error(`${viewName} did not expand properly, expected normalized relation "${v.relationName}" but got ${col.relationName} instead.`); 21 | } 22 | if (col.dataType !== v.dataType) { 23 | logger.error(`${viewName} did not expand properly, expected type "${v.dataType}" but got ${col.dataType} instead.`); 24 | } 25 | }); 26 | } 27 | const q = ` 28 | create event table t (a int, b int); 29 | create event table t2 (a int, c int); 30 | create view v1 as select * from t; 31 | create view v2 as select t2.* from t join t2 on t.a = t2.a; 32 | `; 33 | const logger = GenerateUnitTestErrorLogger("assertAllStar", q); 34 | let ast = ParsePlainDielAst(q); 35 | CompileAst(ast); 36 | assertColumns("v1", [ 37 | {columnName: "a", relationName: "t", dataType: DielDataType.Number}, 38 | {columnName: "b", relationName: "t", dataType: DielDataType.Number} 39 | ]); 40 | assertColumns("v2", [ 41 | {columnName: "a", relationName: "t2", dataType: DielDataType.Number}, 42 | {columnName: "c", relationName: "t2", dataType: DielDataType.Number} 43 | ]); 44 | logger.pass(); 45 | return true; 46 | } 47 | -------------------------------------------------------------------------------- /tests/compilerTests/testAsyncPolicy.ts: -------------------------------------------------------------------------------- 1 | import { DbType } from "../../src"; 2 | import { DielPhysicalExecution } from "../../src/compiler/DielPhysicalExecution"; 3 | import { CompileAst, ParsePlainDielAst } from "../../src/compiler/compiler"; 4 | import { LogicalTimestep, RelationType, Relation } from "../../src/parser/dielAstTypes"; 5 | import { GetAsyncViewNameFromOutput } from "../../src/runtime/asyncPolicies"; 6 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 7 | 8 | // the logic here is primarily in DielPhysicalExecution 9 | // mock up a dielphsyical execution class 10 | 11 | export function testAsyncPolicy() { 12 | const query = ` 13 | REGISTER table flights (origin text, delay integer); 14 | create output o1 as 15 | select distinct origin from flights where delay > 500; 16 | `; 17 | const logger = GenerateUnitTestErrorLogger("testAsyncPolicy", query); 18 | const ast = ParsePlainDielAst(query); 19 | CompileAst(ast); 20 | const physicalMetaData = { 21 | dbs: new Map([[1, {dbType: DbType.Local}], [2, {dbType: DbType.Worker}]]), 22 | relationLocation: new Map([["flights", {dbId: 2}]]) 23 | }; 24 | // dummy! 25 | const getEventByTimestep = (n: LogicalTimestep) => ""; 26 | const addRelationToDielMock = (r: Relation) => {}; 27 | const physicalExecution = new DielPhysicalExecution(ast, physicalMetaData, getEventByTimestep, addRelationToDielMock); 28 | // now assert! 29 | const local = physicalExecution.sqlAstSpecPerDb.get(1); 30 | // local must have o1 as select from an event 31 | const remote = physicalExecution.sqlAstSpecPerDb.get(2); 32 | if (!remote) return logger.error(`Remote was not defined`); 33 | const asyncViewName = GetAsyncViewNameFromOutput("o1"); 34 | const asyncView = remote.relations.find(r => r.rName === asyncViewName); 35 | if (!asyncView) { 36 | logger.error(`remote async view was not generated`); 37 | } 38 | const shippedAsyncView = local.relations.find(r => r.rName === asyncViewName); 39 | if (!shippedAsyncView) { 40 | logger.error(`local async view event was not generated`); 41 | if (!shippedAsyncView.isDynamic) { 42 | logger.error(`local async view event should be marked as dynamic`); 43 | } 44 | } 45 | const outputView = local.relations.find(r => r.rName === "o1"); 46 | if (!outputView) { 47 | logger.error(`local output was not generated`); 48 | } 49 | logger.pass(); 50 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DIEL 2 | 3 | [![Build Status](https://travis-ci.org/yifanwu/diel.svg?branch=master)](https://travis-ci.org/yifanwu/diel) 4 | 5 | We live in an era of big data and rich data visualization. However, as data increase in size, browser-based interactive visualizations eventually hit limits in storage and processing capacity. In order to provide interactivity over large datasets, visualization systems need to be rewritten to make use of powerful back-end services. 6 | 7 | Unfortunately, this puts the burden of back-end expertise onto front-end developers. It would be far preferable if front-end developers could write visualizations once in a natural way, and have a framework take responsibility for transparently scaling up the visualization to use back-end services as needed. Achieving this goal requires rethinking how communication and state are managed by the framework: the mapping of interaction logic to server APIs or database queries, handling of results arriving asynchronously over the network, as well as basic cross-layer performance optimizations like caching. 8 | 9 | We present DIEL, a framework that achieves this cross-layer and transparent autoscaling with a simple, declarative interface. DIEL treats UI events as a stream of data that is captured in an event history for reuse. Developers declare what the state of the interface should be after the arrival of events (user interaction, asynchronous network messages, data arrival). DIEL compiles these declarative specifications into relational queries over both event history and the data to be visualized. In doing so, DIEL makes it easier to develop visualizations that are robust against changes to the size and location of data. To evaluate the DIEL framework, we developed a prototype implementation and confirmed that DIEL supports a range of visualization and interaction designs. Further, visualizations written using DIEL can transparently and seamlessly scale to use back-end services with little intervention from the developer. 10 | 11 | ## Team 12 | 13 | DIEL is a research project at [UC Berkeley RISE Lab](https://rise.cs.berkeley.edu/), by [Yifan Wu](http://twitter.com/yifanwu), with advising from [Joe Hellerstein](http://twitter.com/joe_hellerstein), [Eugene Wu](http://twitter.com/sirrice) and [Remco Chang](http://www.cs.tufts.edu/~remco/), and help from undergrad researchers [Ryan Purpura](http://github.com/rmpurp) and [Lucie Choi](http://github.com/dkqntiqn). 14 | -------------------------------------------------------------------------------- /tests/compilerTests/testDependency.ts: -------------------------------------------------------------------------------- 1 | import { DerivedRelation, Relation } from "../../src/parser/dielAstTypes"; 2 | import { IsSetIdentical } from "../../src/util/dielUtils"; 3 | import { ParsePlainDielAst } from "../../src/compiler/compiler"; 4 | import { GetOriginalRelations, GetAllDerivedViews } from "../../src/compiler/DielAstGetters"; 5 | import { DeriveOriginalRelationsAViewDependsOn, AddDepTree } from "../../src/compiler/passes/dependency"; 6 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 7 | 8 | export function testGetOriginalRelationsDependedOn() { 9 | const logger = GenerateUnitTestErrorLogger("testGetOriginalRelationsDependedOn", q1); 10 | let ast = ParsePlainDielAst(q1); 11 | const views = GetAllDerivedViews(ast); 12 | const deps = AddDepTree(ast); 13 | 14 | // @LUCIE TODO: make assertions about the dep tree here 15 | if ((!deps.get("v1")) || deps.get("v1").dependsOn[0] !== "t1") { 16 | logger.error(`V1 dependency incorrect!`); 17 | } 18 | 19 | let originalRelations = [] as string[]; 20 | GetOriginalRelations(ast).forEach(function(value: Relation) { 21 | originalRelations.push(value.rName); 22 | }); 23 | 24 | views.forEach(function(view: DerivedRelation) { 25 | let dependedTables = DeriveOriginalRelationsAViewDependsOn(deps, view.rName); 26 | if (!IsSetIdentical(answer.get(view.rName), dependedTables)) { 27 | logger.error(`Two sets are not the same, with the dependency tree`, [deps, {expected: answer.get(view.rName), got: dependedTables}]); 28 | } 29 | }); 30 | logger.pass(); 31 | } 32 | 33 | let answer = new Map>(); 34 | answer.set("v1", new Set(["t1"])); 35 | answer.set("v2", new Set(["t1"])); 36 | answer.set("v3", new Set(["t1", "t2"])); 37 | answer.set("v4", new Set(["t1", "t2"])); 38 | answer.set("v5", new Set(["t1", "t2"])); 39 | answer.set("v6", new Set(["t1", "t2"])); 40 | 41 | 42 | /** 43 | * 44 | t1 t2 45 | | \ / 46 | v1 v3 47 | \ \ 48 | v2 v4 t3 49 | / \ / 50 | v6 v5 51 | */ 52 | let q1 = 53 | ` 54 | create event table t1 (a integer); 55 | create event table t2 (b integer); 56 | register table t3 (c integer); 57 | 58 | create view v1 as select * from t1; 59 | create view v2 as select * from v1; 60 | create view v3 as select * from t1 join t2 on t1.a = t2.b; 61 | create view v4 as select * from v3; 62 | create view v5 as select * from v4, t3; 63 | create output v6 as select * from v4; 64 | `; -------------------------------------------------------------------------------- /tests/parserTests/basicOperatorsTest.ts: -------------------------------------------------------------------------------- 1 | import { ParsePlainDielAst } from "../../src/compiler/compiler"; 2 | import { assertExprAsFunctionWithName, assertExprAsColumnWithname, assertValue, GenerateUnitTestErrorLogger } from "../testHelper"; 3 | import { ExprColumnAst, ExprFunAst, DerivedRelation } from "../../src/parser/dielAstTypes"; 4 | import { GetRelationDef } from "../../src/compiler/DielAstGetters"; 5 | 6 | export function assertBasicOperators() { 7 | const q = ` 8 | create event table Attendance ( 9 | arrival int, 10 | departure int, 11 | aid int 12 | ); 13 | create view v1 as select avg(arrival), aid from Attendance group by aid having avg(arrival) > 8 order by aid asc limit 5; 14 | `; 15 | const logger = GenerateUnitTestErrorLogger("assertBasicOperators", q); 16 | const ast = ParsePlainDielAst(q); 17 | const v1Relation = (GetRelationDef(ast, "v1") as DerivedRelation).selection.compositeSelections[0].relation; 18 | // test group by 19 | const groupByClauses = v1Relation.groupByClause; 20 | if (groupByClauses.selections.length !== 1) { 21 | logger.error(`View v1 does not have group by clause!`); 22 | } 23 | const groupByColumnName = (groupByClauses.selections[0]).columnName; 24 | if (groupByColumnName !== "aid") { 25 | logger.error(`View v1 does not have the expected group by clause of aid, instead it got ${groupByColumnName}`); 26 | } 27 | // test having 28 | if (!assertExprAsFunctionWithName(groupByClauses.predicate, ">")) { 29 | logger.error(`View v1's having clause didn't recognize the function expression, got ${JSON.stringify(groupByClauses.predicate)} instead`); 30 | } 31 | const avgExpr = (groupByClauses.predicate).args[0]; 32 | if (!assertExprAsFunctionWithName(avgExpr, "avg")) { 33 | logger.error(`View v1's having clause didn't recognize the column argument, got ${JSON.stringify(avgExpr)} instead`); 34 | } 35 | // test order by 36 | const orderByClauses = v1Relation.orderByClause; 37 | if (!assertExprAsColumnWithname(orderByClauses[0].selection, "aid")) { 38 | logger.error(`View v1's having clause didn't recognize the column argument, got ${JSON.stringify(orderByClauses[0].selection)} instead`); 39 | } 40 | // test limit by 41 | const limitClause = v1Relation.limitClause; 42 | if (!assertValue(limitClause, 5)) { 43 | logger.error(`View v1's limit clause didn't get 5 properly, got ${JSON.stringify(limitClause)} instead`); 44 | } 45 | logger.pass(); 46 | return true; 47 | } 48 | 49 | -------------------------------------------------------------------------------- /tests/index.ts: -------------------------------------------------------------------------------- 1 | import { assertSimpleType, assertMultiplyType } from "./compilerTests/assertTypes"; 2 | import { testTopologicalSort, testDistributionLogc } from "./unitTest"; 3 | import { assertBasicNormalizationOfRelation } from "./compilerTests/assertNormalization"; 4 | import { assertBasicOperators } from "./parserTests/basicOperatorsTest"; 5 | import { assertAllStar } from "./compilerTests/testStarExpantion"; 6 | import { assertBasicConstraints } from "./parserTests/constraintsTest"; 7 | import { assertFunctionParsing } from "./parserTests/functionTest"; 8 | import { assertLatestSyntax } from "./compilerTests/testSyntaxSugar"; 9 | import { codeGenBasicSQLTest } from "./sqlCodeGenTest"; 10 | import { testTriTableCreation } from "./testEventTableCacheCreation"; 11 | import { testGetOriginalRelationsDependedOn } from "./compilerTests/testDependency"; 12 | import { assertCheckViewConstraintTest } from "./compilerTests/testViewConstraints"; 13 | import { ParsePlainDielAst, CompileAst } from "../src/compiler/compiler"; 14 | import { testAsyncPolicy } from "./compilerTests/testAsyncPolicy"; 15 | import { testMaterializationRuntime } from "./compilerTests/testMaterializationRuntime"; 16 | // import { PrintCode } from "../src/util/messages"; 17 | testTriTableCreation(); 18 | testAsyncPolicy(); 19 | assertLatestSyntax(); 20 | testAsyncPolicy(); 21 | testTopologicalSort(); 22 | testGetOriginalRelationsDependedOn(); 23 | testDistributionLogc(); 24 | codeGenBasicSQLTest(); 25 | assertBasicOperators(); 26 | assertSimpleType(); 27 | assertAllStar(); 28 | assertMultiplyType(); 29 | 30 | const q = ` 31 | create event table t1 ( 32 | a int, 33 | b int 34 | ); 35 | create event table t2 ( 36 | c text, 37 | b int 38 | ); 39 | 40 | REGISTER UDF testAdd TYPE int; 41 | 42 | create view v1 as select a from t1 join t2 on t1.b = t2.b where c = 'cat'; 43 | create view v2 as select a from t1 join (select max(b) as b from t2) m on m.b = t1.b; 44 | create view v3 as select a from t1 where b in (select b from t2 where c = 'hello'); 45 | create view v5 as select testAdd(a, b) from t1; 46 | create view v4 as select datetime(a, 'unixepoch') from t1; 47 | `; 48 | 49 | let ast = ParsePlainDielAst(q); 50 | CompileAst(ast); 51 | assertBasicNormalizationOfRelation(ast, q); 52 | assertFunctionParsing(ast, q); 53 | 54 | // @LUCIE: the following tests are failing, fix me 55 | // assertBasicConstraints(); 56 | // assertCheckViewConstraintTest(); 57 | 58 | // @LUCIE: the following tests are not defined: 59 | // testMaterializedViewConstraint(); 60 | // testMaterialization(); -------------------------------------------------------------------------------- /src/compiler/passes/normalizeAlias.ts: -------------------------------------------------------------------------------- 1 | import { DielAst, RelationReference, RelationReferenceType, RelationReferenceDirect, SelectionUnit, ExprAst, ExprType, ExprColumnAst, ExprFunAst, ExprValAst, DerivedRelation } from "../../parser/dielAstTypes"; 2 | import { ReportDielUserError, ReportDielUserWarning, LogInternalError, DielInternalErrorType } from "../../util/messages"; 3 | import { WalkThroughRelationReferences, WalkThroughSelectionUnits } from "../DielAstVisitors"; 4 | 5 | function relationReferenceVisitor(r: RelationReference) { 6 | if (r.alias) return; 7 | switch (r.relationReferenceType) { 8 | case RelationReferenceType.Subquery: 9 | ReportDielUserError(`Subqueries must be named!`); 10 | return; 11 | case RelationReferenceType.Direct: 12 | r.alias = (r as RelationReferenceDirect).relationName; 13 | } 14 | } 15 | 16 | function visitSelections (r: SelectionUnit) { 17 | r.columnSelections.map(c => { 18 | if (c.alias) return; 19 | c.alias = getAliasForExpr(c.expr); 20 | }); 21 | } 22 | 23 | export function NormalizeAliasForDerivedRelation(derived: DerivedRelation) { 24 | derived.selection.compositeSelections.map(c => { 25 | visitSelections(c.relation); 26 | if (c.relation.baseRelation) { 27 | relationReferenceVisitor(c.relation.baseRelation); 28 | if (c.relation.joinClauses) { 29 | c.relation.joinClauses.map(j => { 30 | relationReferenceVisitor(j.relation); 31 | }); 32 | } 33 | } 34 | }); 35 | } 36 | 37 | /** 38 | * Need to normalize the names for the relation references 39 | * also for column names 40 | * @param ast 41 | */ 42 | export function NormalizeAlias(ast: DielAst) { 43 | // walk through all the Relation References 44 | WalkThroughRelationReferences(ast, relationReferenceVisitor); 45 | WalkThroughSelectionUnits(ast, visitSelections); 46 | return; 47 | } 48 | 49 | function getAliasForExpr(expr: ExprAst): string | null { 50 | switch (expr.exprType) { 51 | case ExprType.Column: 52 | return (expr as ExprColumnAst).columnName; 53 | case ExprType.Func: 54 | ReportDielUserWarning(`Should name func column!`); 55 | return (expr as ExprFunAst).functionReference; 56 | case ExprType.Val: 57 | ReportDielUserWarning(`Should name value column!`); 58 | const v = (expr as ExprValAst); 59 | return v.dataType + v.value; 60 | case ExprType.Star: 61 | return null; 62 | default: 63 | return LogInternalError(`Not all ${expr.exprType} handled`, DielInternalErrorType.UnionTypeNotAllHandled); 64 | } 65 | } -------------------------------------------------------------------------------- /testEndToEnd/studentTest.ts: -------------------------------------------------------------------------------- 1 | import * as path from "path"; 2 | import { DielRuntime, DbSetupConfig, DbType, RelationObject } from "../src"; 3 | import { LogInternalError, LogTest, LogInternalWarning } from "../src/util/messages"; 4 | 5 | const jsFile = path.resolve(__dirname, "../../..//node_modules/sql.js/dist/worker.sql-wasm.js"); 6 | 7 | const dbConfigs: DbSetupConfig[] = [{ 8 | dbType: DbType.Worker, 9 | jsFile, 10 | dataFile: path.resolve(__dirname, "../../testEndToEnd/data/students.sqlite") 11 | }, 12 | ]; 13 | 14 | // data in students.sqlite 15 | // students 16 | // Alice|1 17 | // Bob|2 18 | // Charlie|3 19 | // Dan|4 20 | // Ellis|5 21 | // exam 22 | // 1|50 23 | // 2|85 24 | // 3|99 25 | // 4|12 26 | // 5|100 27 | 28 | const mainDbPath: string = null; 29 | const dielFiles = [path.resolve(__dirname, "../../testEndToEnd/diel/students.diel")]; 30 | const isCaching = true; 31 | 32 | const answerByRequestTimestep = new Map(); 33 | // timesteps start on 2 since the first 1 is used at setup time 34 | answerByRequestTimestep.set("Alice", 50); 35 | answerByRequestTimestep.set("Bob", 85); 36 | answerByRequestTimestep.set("Charlie", 99); 37 | 38 | export function testStudentDb(perf: (diel: DielRuntime) => void) { 39 | 40 | const diel = new DielRuntime({ 41 | isStrict: true, 42 | showLog: true, 43 | setupCb: testClass, 44 | caching: isCaching, 45 | dielFiles, 46 | mainDbPath, 47 | dbConfigs, 48 | }); 49 | 50 | async function testClass() { 51 | console.log(`Cache test with${isCaching ? "" : " no"} caching starting`); 52 | 53 | diel.BindOutput("grade_result", (o: RelationObject) => { 54 | LogTest("results!", JSON.stringify(o)); 55 | if (o.length !== 1) { 56 | LogInternalError(`We expected only one result`); 57 | } 58 | const result = o[0]["grade"]; 59 | const answer = answerByRequestTimestep.get(o[0]["student"] as string); 60 | if (result !== answer) { 61 | LogInternalWarning(`We expected ${answer} but got ${result} instead`); 62 | } 63 | }); 64 | 65 | diel.NewInput("name_choice", {first_name: "Alice"}); 66 | // window.setTimeout(() => { 67 | diel.NewInput("name_choice", {first_name: "Bob"}); 68 | diel.NewInput("name_choice", {first_name: "Charlie"}); 69 | diel.NewInput("name_choice", {first_name: "Alice"}); 70 | diel.NewInput("name_choice", {first_name: "Charlie"}); 71 | // }, 1000); 72 | // window.setTimeout(() => { 73 | // diel.NewInput("name_choice", {first_name: "Alice"}); 74 | // }, 1000); 75 | // manually wait for like 5 seconds and it should be enough time 76 | window.setTimeout(() => { 77 | perf(diel); 78 | }, 5000); 79 | } 80 | return diel; 81 | } 82 | -------------------------------------------------------------------------------- /tests/parserTests/constraintsTest.ts: -------------------------------------------------------------------------------- 1 | import { ParsePlainDielAst, CompileAst } from "../../src/compiler/compiler"; 2 | import { ExprFunAst, ExprType, ExprParen, OriginalRelation } from "../../src/parser/dielAstTypes"; 3 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 4 | import { GetRelationDef } from "../../src/compiler/DielAstGetters"; 5 | 6 | export function assertBasicConstraints() { 7 | const q = ` 8 | CREATE EVENT TABLE Persons ( 9 | Name text, 10 | PersonID int NOT NULL, 11 | Age int, 12 | CHECK (Age>=18) 13 | ); 14 | 15 | CREATE TABLE Orders ( 16 | OrderID int NOT NULL, 17 | OrderNumber int NOT NULL, 18 | PersonID int, 19 | PRIMARY KEY (OrderID), 20 | FOREIGN KEY (PersonID) REFERENCES Persons(PersonID) 21 | );`; 22 | let ast = ParsePlainDielAst(q); 23 | CompileAst(ast); 24 | const logger = GenerateUnitTestErrorLogger("assertBasicConstraints", q); 25 | const ordersTable = GetRelationDef(ast, "Orders") as OriginalRelation; 26 | if (!ordersTable) { 27 | logger.error(`Did not even parse Orders table`); 28 | } 29 | console.log(JSON.stringify(ordersTable)); 30 | 31 | // foreign key 32 | const fk = ordersTable.constraints.foreignKeys; 33 | if (!fk) { 34 | logger.error(`foreign key not created`); 35 | } 36 | const expected = "PersonID"; 37 | if ((fk[0].sourceColumn !== expected)) { 38 | logger.error(`created foreign key source column is wrong, got ${fk[0].sourceColumn} but expected ${expected}`); 39 | } 40 | if (fk[0].targetColumn !== expected) { 41 | logger.error(`created foreign key targetColumn is wrong, got ${fk[0].targetColumn}`); 42 | } 43 | if (fk[0].targetRelation !== "Persons") { 44 | logger.error(`created foreign key targetRelation is wrong, got ${fk[0].targetRelation}`); 45 | } 46 | // NOT NULL 47 | const notNulls = ordersTable.constraints.notNull; 48 | if (notNulls.length !== 2) { 49 | logger.error(`Missing not nulls, only have ${JSON.stringify(notNulls)}`); 50 | } 51 | // a bit brittle here, assumes parsing order 52 | if (notNulls[0] !== "OrderID") { 53 | logger.error(`Parsed wrong not null; have ${notNulls[0]} but expects OrderID`); 54 | } 55 | if (notNulls[1] !== "OrderNumber") { 56 | logger.error(`Parsed wrong not null; have ${JSON.stringify(notNulls[1])}`); 57 | } 58 | // Primary key 59 | const pk = ordersTable.constraints.primaryKey; 60 | if ((!pk) || pk[0] !== "OrderID") { 61 | logger.error(`Primary key not created or is wrong, ${pk}`); 62 | } 63 | // check 64 | const personsTable = GetRelationDef(ast, "Persons") as OriginalRelation; 65 | const checks = personsTable.constraints.exprChecks; 66 | if ((!checks) || checks.length === 0) { 67 | logger.error(`check not created, ${JSON.stringify(checks)}`); 68 | } 69 | if (checks[0].exprType !== ExprType.Parenthesis || (((checks[0]).content)).functionReference !== ">=") { 70 | logger.error(`check not correct, ${JSON.stringify(checks[0])}`); 71 | } 72 | logger.pass(); 73 | return true; 74 | } -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "enable": true, 3 | "linterOptions": { 4 | "exclude": [ 5 | "src/parser/grammar/*.ts", 6 | "node_modules/**" 7 | ] 8 | }, 9 | "jsRules": { 10 | "class-name": true, 11 | "comment-format": [ 12 | true, 13 | "check-space" 14 | ], 15 | "indent": [ 16 | true, 17 | "spaces" 18 | ], 19 | "ordered-imports": true, 20 | "no-duplicate-variable": true, 21 | "no-eval": true, 22 | "no-trailing-whitespace": true, 23 | "no-unsafe-finally": true, 24 | "one-line": [ 25 | true, 26 | "check-open-brace", 27 | "check-whitespace" 28 | ], 29 | "quotemark": [ 30 | true, 31 | "double" 32 | ], 33 | "semicolon": [ 34 | true, 35 | "always" 36 | ], 37 | "triple-equals": [ 38 | true, 39 | "allow-null-check" 40 | ], 41 | "variable-name": [ 42 | true, 43 | "ban-keywords" 44 | ], 45 | "whitespace": [ 46 | true, 47 | "check-branch", 48 | "check-decl", 49 | "check-operator", 50 | "check-separator", 51 | "check-type" 52 | ] 53 | }, 54 | "rules": { 55 | "class-name": true, 56 | "comment-format": [ 57 | true, 58 | "check-space" 59 | ], 60 | "indent": [ 61 | true, 62 | "spaces" 63 | ], 64 | "no-eval": true, 65 | "no-internal-module": true, 66 | "no-trailing-whitespace": true, 67 | "no-unsafe-finally": true, 68 | "no-var-keyword": true, 69 | "one-line": [ 70 | true, 71 | "check-open-brace", 72 | "check-whitespace" 73 | ], 74 | "quotemark": [ 75 | true, 76 | "double" 77 | ], 78 | "semicolon": [ 79 | true, 80 | "always" 81 | ], 82 | "triple-equals": [ 83 | true, 84 | "allow-null-check" 85 | ], 86 | "typedef-whitespace": [ 87 | true, 88 | { 89 | "call-signature": "nospace", 90 | "index-signature": "nospace", 91 | "parameter": "nospace", 92 | "property-declaration": "nospace", 93 | "variable-declaration": "nospace" 94 | } 95 | ], 96 | "variable-name": [ 97 | true, 98 | "ban-keywords" 99 | ], 100 | "whitespace": [ 101 | true, 102 | "check-branch", 103 | "check-decl", 104 | "check-operator", 105 | "check-separator", 106 | "check-type" 107 | ] 108 | } 109 | } -------------------------------------------------------------------------------- /src/compiler/DielAstVisitors.ts: -------------------------------------------------------------------------------- 1 | import { DielAst, Relation, AstType, InsertionClause, SelectionUnit, RelationReference, RelationNameType } from "../parser/dielAstTypes"; 2 | import { LogInternalError, LogInternalWarning } from "../util/messages"; 3 | import { DerivedRelation } from ".."; 4 | import { GetAllDerivedViews, IsRelationTypeDerived } from "./DielAstGetters"; 5 | import { AddSingleDependencyByDerivedRelation } from "./passes/dependency"; 6 | 7 | type SelectionUnitFunction = (s: SelectionUnit, ast?: DielAst, relationName?: string) => T; 8 | 9 | // --------------- BEGIN SETTERS -------------------- 10 | 11 | export function AddRelation(ast: DielAst, newR: Relation) { 12 | // make sure that the name is not found 13 | const idx = ast.relations.findIndex(r => r.rName === newR.rName); 14 | if (idx > -1) { 15 | LogInternalError(`The relation you want to add, ${newR.rName}, is already defined`); 16 | } else { 17 | ast.relations.push(newR); 18 | // also need to update the dependency tree.... 19 | if (IsRelationTypeDerived(newR.relationType)) { 20 | AddSingleDependencyByDerivedRelation(ast, newR as DerivedRelation); 21 | } 22 | } 23 | } 24 | 25 | 26 | export function DeleteRelation(ast: DielAst, relationName: RelationNameType) { 27 | const idx = ast.relations.findIndex(r => r.rName === relationName); 28 | if (idx > -1) { 29 | return ast.relations.splice(idx, 1); 30 | } 31 | LogInternalWarning(`Relation ${relationName} to delete was not found`); 32 | // also need to update the dependency tree.... 33 | if (ast.depTree) { 34 | ast.depTree.delete(relationName); 35 | } 36 | return null; 37 | } 38 | 39 | // --------------- BEGIN VISITORS -------------------- 40 | 41 | // we are going to assume that the AST is sorted in topological order already 42 | export function WalkThroughSelectionUnits(ast: DielAst, fun: SelectionUnitFunction): T[] { 43 | const results: T[] = []; 44 | function applyToDerivedRelation(r: DerivedRelation, fun: SelectionUnitFunction): void { 45 | r.selection.compositeSelections.map(c => { 46 | const result = fun(c.relation, ast, r.rName); 47 | results.push(result); 48 | }); 49 | } 50 | const derived = GetAllDerivedViews(ast); 51 | derived.map(r => { 52 | applyToDerivedRelation(r, fun); 53 | }); 54 | ast.programs.forEach((commands, _) => { 55 | commands.map(c => { 56 | // check the select clause from insert 57 | if (c.astType === AstType.Insert) { 58 | const insertClause = c as InsertionClause; 59 | if (insertClause.selection) { 60 | insertClause.selection.compositeSelections.map(s => fun(s.relation, ast, undefined)); 61 | } 62 | } 63 | }); 64 | }); 65 | return results; 66 | } 67 | 68 | // FIXME: I think there are other places with Relation Reference.. should fix later 69 | export function WalkThroughRelationReferences(ast: DielAst, fun: (r: RelationReference) => T): T[] { 70 | const results: T[] = []; 71 | const selectionUnitFunc: SelectionUnitFunction = (s: SelectionUnit) => { 72 | if (s.baseRelation) { 73 | results.push(fun(s.baseRelation)); 74 | if (s.joinClauses) { 75 | s.joinClauses.map(j => { 76 | results.push(fun(j.relation)); 77 | }); 78 | } 79 | } 80 | }; 81 | WalkThroughSelectionUnits(ast, selectionUnitFunc); 82 | return results; 83 | } -------------------------------------------------------------------------------- /docs/internals.md: -------------------------------------------------------------------------------- 1 | # DIEL Internals 2 | 3 | ## Intermediate Representation of DIEL Programs 4 | 5 | _DIEL internals_: you do __not__ need to read this to use the DIEL framework. 6 | 7 | This document walk through how we initially take in queries in the logical space (DIEL IR), then compile it down to execution logic against easch individual database instance (SQL IR), as well as a runtime with event logic. 8 | 9 | DIEL specifications needs both an event loop and distributed query processing to run. We discussed the event loop in [another post](./events.md), and we discuss the representation of the program representation for each logical step here. 10 | 11 | ### Logical Specification 12 | 13 | DIEL IR have a few extra relation types. 14 | 15 | Since __caching__ is across events, we instrument it in the DIEL IR layer. It's a query rewrite that adds a layer of indirection to the data, and provides an internal structure for garbage collection, which we are looking into implementing as well. 16 | 17 | ### Physical Execution 18 | 19 | To __distribute__ the query processing to different nodes and make sure that the nodes have their respective data sources in order to evaluate the queries. DIEL first generates a basic query plan based on the query AST, it then recurses down the plan to decide whether the view needs to be "shipped". A view needs to be shipped if it is part of a query that spans multiple databases. DIEL picks a “leader” database to execute the query based on the simple heuristic of minimizing the amount of data being shipped. 20 | 21 | Based on this shipping specification, at set up time, DIEL ships the queries to the databases that needs to execute these queries, and at run time, DIEL ships the data to the relevant databases. Each shipment of data has its corresponding request timestep. Each database waits for its dependencies for the lowest timestep in the queue and once the dependencies are met, evaluates the queries, shares them with DIEL (or downstream databases), and then evaluates the next timestep in the queue. 22 | 23 | Once we generate the SQL IR for each node, we then perform __materialized view maintenance__. First, DIEL materializes intermediate views that are used multiple times. Since views are just named queries in SQL, whenever the database evaluates a query that references the views, the view is also evaluated as part of the query. However, this is inefficient if the view is shared by multiple outputs, because the same query is evaluated multiple times even when we know that the results are the same. To fix this problem, the DIEL optimizer rewrites the views to tables and refreshes the values of the tables by running the view query after the dependent event tables change. 24 | Second, DIEL only invokes the rendering function if there is a new dependent event. This can improve performance if the rendering function naively recomputes and populates the DOM elements. 25 | Third, when evaluating async views, DIEL creates a cache based on past results. For every new event, DIEL first hashes the param- eters of the event tables and then looks up the hash in an internal table, request cache, which contains three columns hash, dataId, and viewName. If the hash is present, DIEL inserts the dataId into the corre- sponding event table for the async view. If the hash is not present, DIEL dispatches the events to evaluate the async views and stores the results with the hash once the result has been received. The dataId creates another layer of indirection that saves storage by duplicating “pointers” as opposed to the actual data. These functionalities are implemented via a combination of compile time query rewrites and runtime event handling. 26 | 27 | #### `DbEngines` 28 | 29 | The `DbEngines.ts` file (and class) is a wrapper around each database instance. 30 | 31 | ## Data Structures 32 | 33 | -------------------------------------------------------------------------------- /src/runtime/runtimeTypes.ts: -------------------------------------------------------------------------------- 1 | import { DbIdType, RelationNameType, LogicalTimestep, RelationType, Relation, ToBeFilled } from "../parser/dielAstTypes"; 2 | import { DbSetupConfig } from "./DbEngine"; 3 | import { SqlRelation } from "../parser/sqlAstTypes"; 4 | 5 | export type ExecutionSpec = {dbId: DbIdType, relationDef: SqlRelation}[]; 6 | 7 | export interface NodeDependencyAugmented extends NodeDependency { 8 | remoteId?: ToBeFilled; // only has remoteId if it's an original table 9 | } 10 | 11 | export type NodeDependency = { 12 | relationName: string; 13 | dependsOn: RelationNameType[], 14 | isDependedBy: RelationNameType[], 15 | // the following should probably be refactored, but convenient for now. 16 | isDynamic: boolean; // e.g. event tables are 17 | }; 18 | 19 | // this should work for either SQL or DIEL types 20 | export type DependencyTree = Map; 21 | 22 | export interface DependencyInfo { 23 | // both ways for easy access 24 | depTree: DependencyTree; 25 | topologicalOrder: string[]; 26 | inputDependenciesOutput: Map>; 27 | inputDependenciesAll: Map>; 28 | } 29 | 30 | export interface DielConfig { 31 | dielFiles: string[]; 32 | setupCb: () => void; 33 | showLog?: boolean; 34 | isStrict?: boolean; 35 | mainDbPath?: string; 36 | dbConfigs?: DbSetupConfig[]; 37 | caching?: boolean; 38 | } 39 | 40 | export type GetRelationToShipFuncType = (dbId: DbIdType, relation: string, step: LogicalTimestep) => Set; 41 | 42 | export type RecordObject = {[index: string]: string | number | Uint8Array}; 43 | export type RelationObject = RecordObject[]; 44 | 45 | export enum DbType { 46 | Local = "Local", 47 | Worker = "Worker", 48 | Socket = "Socket" 49 | } 50 | 51 | // TODO: add more runtime info 52 | // e.g., number of times accessed etc. 53 | export interface TableMetaData { 54 | dbId: DbIdType; 55 | } 56 | 57 | export enum DielRemoteAction { 58 | ConnectToDb = "ConnectToDb", 59 | GetResultsByPromise = "GetResultsByPromise", 60 | DefineRelations = "DefineRelations", 61 | UpdateRelation = "UpdateRelation", 62 | ShipRelation = "ShipRelation", 63 | CleanUpQueries = "CleanUpQueries", 64 | Close = "Close" 65 | } 66 | 67 | export interface DielRemoteMessageId { 68 | remoteAction: DielRemoteAction; 69 | relationName?: RelationNameType; 70 | msgId?: number; // currently only used for fullfilling promises. 71 | requestTimestep?: number; 72 | } 73 | export interface DielRemoteReply { 74 | id: DielRemoteMessageId; 75 | results: RelationObject; 76 | err: any; 77 | } 78 | 79 | interface DielRemoteMessageBase { 80 | remoteAction: DielRemoteAction; 81 | requestTimestep: LogicalTimestep; 82 | msgId?: number; 83 | } 84 | 85 | export interface RemoteGetResultsByPromiseMessage extends RemoteExecuteMessage { 86 | msgId: number; 87 | } 88 | 89 | export interface RemoteShipRelationMessage extends DielRemoteMessageBase { 90 | relationName: RelationNameType; 91 | dbId: DbIdType; 92 | } 93 | 94 | export interface RemoteOpenDbMessage extends DielRemoteMessageBase { 95 | message?: string; // for socket 96 | buffer?: Uint8Array; // for worker 97 | } 98 | 99 | export interface RemoteUpdateRelationMessage extends RemoteExecuteMessage { 100 | relationName: RelationNameType; // redundancy 101 | } 102 | 103 | export interface RemoteExecuteMessage extends DielRemoteMessageBase { 104 | sql: string; 105 | } 106 | 107 | 108 | export type DielRemoteMessage = RemoteGetResultsByPromiseMessage 109 | | RemoteShipRelationMessage 110 | | RemoteUpdateRelationMessage 111 | | RemoteOpenDbMessage 112 | ; -------------------------------------------------------------------------------- /src/util/messages.ts: -------------------------------------------------------------------------------- 1 | import { STRICT } from "../runtime/DielRuntime"; 2 | 3 | export const FgRed = "\x1b[31m"; 4 | export const FgGreen = "\x1b[32m"; 5 | export const FgBlue = "\x1b[34m"; 6 | export const FgGray = "\x1b[90m"; 7 | export const Underscore = "\x1b[4m"; 8 | export const FgCyan = "\x1b[36m"; 9 | export const FgMagenta = "\x1b[35m"; 10 | 11 | export const BgRed = "\x1b[41m"; 12 | export const BgGreen = "\x1b[42m"; 13 | export const BgYellow = "\x1b[43m"; 14 | 15 | export const Reset = "\x1b[0m"; 16 | 17 | export enum DielInternalErrorType { 18 | Untitled = "Untitled", 19 | ArgNull = "ArgNull", 20 | TypeError = "TypeError", 21 | RelationNotFound = "RelationNotFound", 22 | NotImplemented = "NotImplemented", 23 | UnionTypeNotAllHandled = "UnionTypeNotAllHandled", 24 | MalFormedAst = "MalFormedAst", 25 | TestError = "TestError" 26 | } 27 | 28 | export enum DielInternalWarningType { 29 | Untitled = "Untitled", 30 | ArgNull = "ArgNull" 31 | } 32 | 33 | export function PrintCode(code: string) { 34 | const codeWithLine = code.split("\n").map((c, i) => `${i + 1}\t${c}`).join("\n"); 35 | console.log(`DIEL Code\n%s}`, codeWithLine); 36 | } 37 | 38 | export function LogSetup(m: string) { 39 | console.log(`${FgMagenta}${m}${Reset}`); 40 | } 41 | 42 | export function LogExecutionTrace(m: string, obj?: any) { 43 | // return null; 44 | console.log(`${FgGray}%s${Reset}`, m, obj ? obj : ""); 45 | } 46 | 47 | export function LogTest(m: string, obj?: any) { 48 | console.log(`${FgCyan}%s${Reset}`, m, obj ? obj : ""); 49 | } 50 | 51 | export function LogInternalError(m: string, errorType = DielInternalErrorType.Untitled): null { 52 | const message = `Error[${errorType}]: ${m}`; 53 | if (STRICT) { 54 | debugger; 55 | throw new Error(message); 56 | } else { 57 | console.log(`${FgRed}${message}${Reset}`); 58 | } 59 | return null; 60 | } 61 | 62 | export function LogInternalWarning(m: string, wariningType = DielInternalWarningType.Untitled) { 63 | console.log(`${FgMagenta}[${wariningType}]%s${Reset}`, m); 64 | } 65 | 66 | export function LogInfo(m: string, obj?: any) { 67 | if (obj) { 68 | console.log(`${FgBlue}%s${Reset}`, m, obj); 69 | } else { 70 | console.log(`${FgBlue}%s${Reset}`, m); 71 | } 72 | } 73 | 74 | export function LogTmp(m: string) { 75 | console.log(`${FgGray}${m}${Reset}`); 76 | } 77 | 78 | export function LogStandout(m: string) { 79 | console.log(`${BgYellow}%s${Reset}`, m); 80 | } 81 | 82 | export function ReportDielBasicParsingError(m: string) { 83 | console.log(`${FgRed}Parsing error from user provided code:\n%s${Reset}`, m); 84 | if (STRICT) throw new Error(); 85 | } 86 | 87 | export function ReportUserRuntimeWarning(m: string): null { 88 | console.log(`${FgMagenta}[User Error]%s${Reset}`, m); 89 | return null; 90 | } 91 | 92 | // note that this is in browser 93 | export function ReportUserRuntimeError(m: string): null { 94 | if (STRICT) throw new Error(m); 95 | console.log(`${BgYellow}[User Error]%s${Reset}`, m); 96 | return null; 97 | } 98 | 99 | // both warning and error 100 | export enum UserErrorType { 101 | UndefinedScale = "UndefinedScale", 102 | } 103 | 104 | // TODO: this should also report the line of the code 105 | // the input should be more structured 106 | export function ReportDielUserError(m: string, q?: string, errorType?: UserErrorType): null { 107 | console.log(`${FgRed}%s${Reset}`, m); 108 | if (STRICT) throw new Error(); 109 | return null; 110 | } 111 | 112 | export function ReportDielUserWarning(m: string, q?: string) { 113 | console.log(`User Warning: ${FgMagenta}%s${Reset}`, m); 114 | if (q) console.log(`\nQuery: ${FgMagenta}%s${Reset}\n`, q); 115 | } 116 | 117 | export function sanityAssert(b: boolean, msg: string) { 118 | if (!b) { 119 | LogInternalError(`Assertion error: ${msg}`); 120 | } 121 | } -------------------------------------------------------------------------------- /src/compiler/compiler.ts: -------------------------------------------------------------------------------- 1 | import { IsRelationTypeDerived } from "./DielAstGetters"; 2 | import { ApplyTemplates, TryToApplyTemplate, TryToCopyRelationSpec } from "./passes/applyTemplate"; 3 | import { AddDepTree, AddSingleDependencyByDerivedRelation, ArrangeInTopologicalOrder } from "./passes/dependency"; 4 | import { NormalizeConstraints, NormalizeConstraintsForSingleOriginalRelation } from "./passes/normalizeConstraints"; 5 | import { NormalizeColumnSelection, NormalizeColumnForDerivedRelation } from "./passes/normalizeColumnSelection"; 6 | import { InferType, InferTypeForDerivedRelation } from "./passes/inferType"; 7 | import { DerivedRelation, OriginalRelation, DielAst } from "../parser/dielAstTypes"; 8 | import { AddRelation } from "./DielAstVisitors"; 9 | import { NormalizeAlias, NormalizeAliasForDerivedRelation } from "./passes/normalizeAlias"; 10 | 11 | import { ANTLRInputStream, CommonTokenStream } from "antlr4ts"; 12 | import * as parser from "../parser/grammar/DIELParser"; 13 | import * as lexer from "../parser/grammar/DIELLexer"; 14 | import Visitor from "../parser/generateAst"; 15 | import { RelationSelection } from "../parser/dielAstTypes"; 16 | 17 | export function parse(code: string) { 18 | const inputStream = new ANTLRInputStream(code); 19 | const l = new lexer.DIELLexer(inputStream); 20 | const tokenStream = new CommonTokenStream(l); 21 | return new parser.DIELParser(tokenStream); 22 | } 23 | 24 | export function ParsePlainSelectQueryAst(code: string) { 25 | const inputStream = new ANTLRInputStream(code); 26 | const l = new lexer.DIELLexer(inputStream); 27 | const tokenStream = new CommonTokenStream(l); 28 | const p = new parser.DIELParser(tokenStream); 29 | const tree = p.selectQuery(); 30 | let visitor = new Visitor(); 31 | let ast = visitor.visit(tree) as RelationSelection; 32 | return ast; 33 | } 34 | 35 | /** 36 | * this visits all the queries that are included in the string 37 | * @param code code string 38 | */ 39 | export function ParsePlainDielAst(code: string) { 40 | // PrintCode(code); 41 | const inputStream = new ANTLRInputStream(code); 42 | const l = new lexer.DIELLexer(inputStream); 43 | const tokenStream = new CommonTokenStream(l); 44 | const p = new parser.DIELParser(tokenStream); 45 | const tree = p.queries(); 46 | let visitor = new Visitor(); 47 | let ast = visitor.visitQueries(tree); 48 | return ast; 49 | } 50 | 51 | /** 52 | * Note that the compilation here is logical 53 | * there will be another pass on how to do the phyical part 54 | * @param ir the IR that will be manipulated 55 | */ 56 | export function CompileAst(ast: DielAst) { 57 | ApplyTemplates(ast); 58 | NormalizeAlias(ast); 59 | AddDepTree(ast); 60 | ArrangeInTopologicalOrder(ast); 61 | NormalizeConstraints(ast); 62 | NormalizeColumnSelection(ast); 63 | InferType(ast); 64 | return ast; 65 | } 66 | 67 | // there should be a progressive version of this already. 68 | // do not support crossfilter for now, but easy to fix! 69 | export function CompileNewRelationToExistingAst(ast: DielAst, relation: OriginalRelation | DerivedRelation) { 70 | if (IsRelationTypeDerived(relation.relationType)) { 71 | CompileDerivedAstGivenAst(ast, relation as DerivedRelation); 72 | } else { 73 | CompileOriginalAstGivenIr(ast, relation as OriginalRelation); 74 | } 75 | } 76 | 77 | export function CompileDerivedAstGivenAst(ast: DielAst, view: DerivedRelation) { 78 | AddRelation(ast, view); 79 | TryToApplyTemplate(view.selection); 80 | NormalizeAliasForDerivedRelation(view); 81 | AddSingleDependencyByDerivedRelation(ast, view); 82 | NormalizeColumnForDerivedRelation(ast, view); 83 | InferTypeForDerivedRelation(ast, view); 84 | return view; 85 | } 86 | 87 | /** 88 | * TODO: we need to handled the original like derived if they have subqueries... 89 | * @param ast 90 | * @param original 91 | */ 92 | export function CompileOriginalAstGivenIr(ast: DielAst, original: OriginalRelation) { 93 | TryToCopyRelationSpec(ast, original); 94 | // TODO: need to add the depends on dependency... 95 | NormalizeConstraintsForSingleOriginalRelation(original); 96 | return original; 97 | } -------------------------------------------------------------------------------- /tests/compilerTests/testSyntaxSugar.ts: -------------------------------------------------------------------------------- 1 | import { ParsePlainDielAst, ParsePlainSelectQueryAst } from "../../src/compiler/compiler"; 2 | import { DielAst, SelectionUnit } from "../../src/parser/dielAstTypes"; 3 | import { applyLatestToAst, applyLatestToSelectionUnit } from "../../src/compiler/passes/syntaxSugar"; 4 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 5 | import { TestLogger } from "../testTypes"; 6 | import { SqlStrFromSelectionUnit } from "../../src/compiler/codegen/codeGenSql"; 7 | 8 | export function assertLatestSyntax() { 9 | const logger = GenerateUnitTestErrorLogger("assertLatestSyntax"); 10 | for (let test of tests) { 11 | const qAst = ParsePlainSelectQueryAst(test.query); 12 | const aAst = ParsePlainSelectQueryAst(test.answer); 13 | const unit = qAst.compositeSelections[0].relation; 14 | applyLatestToSelectionUnit(unit); 15 | compareAST(aAst.compositeSelections[0].relation, unit, logger); 16 | } 17 | logger.pass(); 18 | } 19 | 20 | /** 21 | */ 22 | function compareAST(ast1: SelectionUnit, ast2: SelectionUnit, logger: TestLogger) { 23 | // Note that relying on JSON stringify is too brittle 24 | // but we can use our own ast to sql string functions! 25 | const q1 = SqlStrFromSelectionUnit(ast1).replace(/\s+/g, ""); 26 | const q2 = SqlStrFromSelectionUnit(ast2).replace(/\s+/g, ""); 27 | // let pretty1 = JSON.stringify(ast1, null, 2); 28 | // let pretty2 = JSON.stringify(ast2, null, 2); 29 | 30 | if (q1 !== q2) { 31 | logger.error(`\n${q1}\n\nis not the same as\n\n${q2}.`); 32 | } 33 | } 34 | 35 | 36 | let tests = [ 37 | // in case of joins, latest should be applied to immediate relations 38 | { 39 | query: ` 40 | select a 41 | from latest t1 42 | join t2 on t1.b = t2.b;`, 43 | answer: ` 44 | select a 45 | from t1 46 | join t2 on t1.b = t2.b 47 | where t1.timestep = (select max(timestep) as timestep from t1);` 48 | }, 49 | // 1. basic 50 | { 51 | query: `select arrival from LATEST t1;`, 52 | answer: `select arrival from t1 where t1.timestep = (select max(timestep) as timestep from t1);`, 53 | }, 54 | // 2. where clause should be preserved 55 | { 56 | query: `select arrival from LATEST t1 where arrival > 10 and arrival < 20;`, 57 | answer: `select arrival from t1 where arrival > 10 and arrival < 20 and t1.timestep = (select max(timestep) as timestep from t1);`, 58 | }, 59 | // 3. constraints, group by, order by, limit should also be preservered 60 | { 61 | query: ` 62 | select count(*), arrival from LATEST t1 63 | where arrival > 10 64 | group by arrival 65 | order by count DESC 66 | limit 10 67 | constrain check (arrival > 10);`, 68 | answer: ` 69 | select count(*), arrival from t1 70 | where arrival > 10 71 | and t1.timestep = (select max(timestep) as timestep from t1) 72 | group by arrival 73 | order by count DESC 74 | limit 10 75 | constrain check (arrival > 10);`, 76 | }, 77 | // 5. check multiple latest for explicit join 78 | { 79 | query: ` 80 | select a 81 | from latest t1 82 | join latest t2 on t1.b = t2.b;`, 83 | 84 | answer: ` 85 | select a 86 | from t1 join t2 on t1.b = t2.b 87 | where t1.timestep = (select max(timestep) as timestep from t1) 88 | and t2.timestep = (select max(timestep) as timestep from t2);`, 89 | 90 | }, 91 | { 92 | 93 | // 6. check multiple latest for implicit join 94 | query: ` 95 | select a 96 | from latest t1, latest t2 97 | where t1.b = t2.b;`, 98 | 99 | answer: ` 100 | select a 101 | from t1, t2 102 | where t1.b = t2.b 103 | and t1.timestep = (select max(timestep) as timestep from t1) 104 | and t2.timestep = (select max(timestep) as timestep from t2);`, 105 | 106 | }, 107 | { 108 | 109 | // 7. check multiple tables 110 | query: ` 111 | select a 112 | from latest t1, t2 113 | where t1.b = t2.b ;`, 114 | 115 | answer: ` 116 | select a 117 | from t1, t2 118 | where t1.b = t2.b 119 | and t1.timestep = (select max(timestep) as timestep from t1);`, 120 | } 121 | ]; 122 | 123 | -------------------------------------------------------------------------------- /docs/events.md: -------------------------------------------------------------------------------- 1 | # Events and Changes in DIEL 2 | 3 | In DIEL everything is "static" within a single logical timestep. 4 | 5 | ## Even Table 6 | 7 | `Event Table`s drive the changes that happen on the screen. For instance, in the [counter example](./index.md#counter-example) we presented, the event table is `clickEvent`, whose changes _drive_ the advance of the logic time. 8 | 9 | ![slider](https://i.ibb.co/RHk6KQT/slider.png) 10 | 11 | Similarly, in the example of a slider, the events that drive changes are the slide interaction, which we define via the following 12 | 13 | ```sql 14 | CREATE EVENT TABLE slideItx(year int) 15 | ``` 16 | 17 | And the event changes the state of the application by the following 18 | 19 | ```sql 20 | CREATE OUTPUT distData AS 21 | SELECT airport, count(*) 22 | FROM flights 23 | JOIN LATEST slideItx 24 | ON year 25 | GROUP BY airport; 26 | ``` 27 | 28 | When there is a new slide interaction, the following happesn 29 | 30 | 1. an event handler in the visualization extracts the slider position and uses a DIEL API to insert it into the slideItx table as a new event record (`{year:2000}`). These events may come from user interactions, automated processes in the client that gener- ate events on behalf of the user (e.g., timers), or from DIEL managed async view evaluations. 31 | 2. Clock Increment. To help manage concurrency, DIEL ensures a global ordering of events in the visualization. To do so, DIEL maintains a logical “clock” that increments whenever a new event is inserted. DIEL will automatically add the current logical timestep as the timestep attribute to new events. For instance, the new slider event is annotated with the timestep 3. 32 | 3. Query Evaluation. DIEL queries are SQL queries with syntactic shortcuts. If the data is in the client, DIEL simply executes the query and passes the result to the visualization for rendering. However if the data is remote (e.g., the flights Table), DIEL automatically sends the necessary data (e.g., the new event record) to the remote database to execute the query, and updates the local output distData table with the result. Notice that the result rows are annotated with both the request timestep (e.g., 3) and the logical timestep when the response arrived (e.g., 4); this provides necessary information to define the developer’s concurrency policy. For instance, responses for other requests (e.g., previous slider events) may arrive out of order, and the DIEL query can select the desired output based on the timesteps. 33 | 4. Output Rendering. DIEL evaluates the output views and invokes the visualization’s rendering function with the result. The function should update the UI (e.g., the bar chart). The rendering function can be implemented with any number of existing frameworks, such as Vega-Lite or D3. 34 | 35 | ## Async View 36 | 37 | DIEL ensures that each event is handled atomically, in a single syn- chronous iteration of the event loop associated with a single logical time. To prevent blocking the UI, it is critical that each iteration runs fast, despite that fact that DIEL queries may be running unpredictably on a remote server. DIEL achieves this by dispatching long-running queries to concurrent services (e.g., remote databases or local worker threads). These services can asynchronously process the queries and return the results at any time, appropriately labeled with request and response timestamps to capture the lag in processing. This is captured in the following diagram. 38 | 39 | ![remote](https://i.ibb.co/zVCQCqq/model3.png) 40 | 41 | Asynchronous results however require direct programmer intervention because they might want to reason with what results get rendered at the current timestep. To address this, we introduce the syntax `async view`, which is evaluated to a relation with logical timesteps that indicate when it was recied and step that tracks what originating timestep it was from. 42 | 43 | For instance, the original output relation `disData` is actually expanded to the following, where `distDataEvent` has the DIEL augmented columns `timestep` and `request_timestep` in addition to its original `airport`, and `count`, which the developer can just query as normal data. 44 | 45 | ```sql 46 | CREATE ASYNC VIEW distDataEvent AS 47 | SELECT origin, COUNT() 48 | FROM flights 49 | JOIN LATEST slideItx ON year 50 | GROUP BY origin; 51 | 52 | CREATE OUTPUT distData AS 53 | SELECT * 54 | FROM distDataEvent e 55 | JOIN LATEST slideItx i ON i.timestep = e.request_timestep; 56 | ``` 57 | -------------------------------------------------------------------------------- /src/runtime/runtimeHelper.ts: -------------------------------------------------------------------------------- 1 | // import { Database, QueryResults } from "sql.js"; 2 | import { RelationObject } from "./runtimeTypes"; 3 | import { RelationSelection, ExprAst, ExprType, ExprColumnAst } from "../parser/dielAstTypes"; 4 | import { LogInternalError } from "../util/messages"; 5 | 6 | type QueryResults = any; 7 | type Database = any; 8 | 9 | function getNameFromExpr(e: ExprAst): string | undefined { 10 | switch (e.exprType) { 11 | case ExprType.Column: 12 | return (e as ExprColumnAst).columnName; 13 | default: 14 | return ""; 15 | } 16 | } 17 | 18 | export function GenerateViewName(q: RelationSelection) { 19 | const hash = Math.floor(Math.random() * 1000).toString(); 20 | if (q.compositeSelections) { 21 | const hintRel = q.compositeSelections[0].relation.baseRelation 22 | ? "_" + q.compositeSelections[0].relation.baseRelation.alias 23 | : ""; 24 | const hintCol = q.compositeSelections[0].relation.columnSelections 25 | .map(c => c.alias ? c.alias : getNameFromExpr(c.expr)) 26 | .join("_"); 27 | return `${hintCol}${hintRel}${hash}`; 28 | } 29 | return hash; 30 | } 31 | 32 | export function CaughtLocalRun(db: Database, s: string) { 33 | try { 34 | console.log(`%c Running Query in Main:\n${s}`, "color: purple"); 35 | db.run(s); 36 | } catch (error) { 37 | LogInternalError(`Error while running\n${s}\n${error}`); 38 | } 39 | } 40 | 41 | /** 42 | * returns all the SQL that defines tables in this DB 43 | * so that we can add to IR parsing (for type inference and other processing needs) 44 | */ 45 | export function getExistingTableDefinitions(isWorker: boolean, db: Database): string { 46 | let queries = ""; 47 | const q = `SELECT name, sql FROM sqlite_master WHERE type='table'`; 48 | 49 | if (isWorker) { 50 | throw new Error(`not yet implemented`); 51 | } else { 52 | const done = () => { console.log("done"); } 53 | db.each(q, (o: any) => { 54 | queries += queries + o.sql; 55 | }, done); 56 | } 57 | return queries; 58 | } 59 | 60 | /** 61 | * Hack: some relations are defined already from another session---currently we don't have a hook to clean up, so skip them 62 | * , relationsToSkip?: string[] 63 | */ 64 | export function processSqlMetaDataFromRelationObject(rO: RelationObject, sourceName: string): string { 65 | // const filtered = (relationsToSkip) 66 | // ? rO.filter(r => !relationsToSkip.find(rS => rS === r["name"])) 67 | // : rO; 68 | return rO.map(definition => definition["sql"].toString() 69 | .replace(/create table/ig, `\n--${sourceName}\nregister table`) + ";") 70 | .join("\n"); 71 | } 72 | 73 | export function ParseSqlJsWorkerResult(data: QueryResults[]): RelationObject { 74 | if (data && (data.length > 0) && data[0].values) { 75 | const o: RelationObject = data[0].values.map((v: any[]) => { 76 | let oi: any = {}; 77 | v.map((vi, i) => { 78 | oi[data[0].columns[i]] = vi; 79 | }); 80 | return oi; 81 | }); 82 | return o; 83 | } 84 | return []; 85 | } 86 | 87 | /** 88 | * emulates better-sqlite's prepare('query').all() 89 | * @param db Database instance 90 | * @param query must be single line with no parameters 91 | */ 92 | export function SqlJsGetObjectArrayFromQuery(db: Database, query: string) { 93 | const stmt = db.prepare(query); 94 | stmt.bind({}); 95 | let r = []; 96 | while (stmt.step()) { 97 | r.push(stmt.getAsObject()); 98 | } 99 | return r; 100 | } 101 | 102 | // /** 103 | // * run time type checker 104 | // * @param fn 105 | // */ 106 | // export const checkType = fn => (params = []) => { 107 | // const { required } = fn; 108 | // const missing = required.filter(param => !(param in params)); 109 | 110 | // if (missing.length) { 111 | // throw new Error(`${ fn.name }() Missing required parameter(s): 112 | // ${ missing.join(", ") }`); 113 | // } 114 | 115 | // return fn(params); 116 | // }; 117 | 118 | 119 | export function convertRelationObjectToQueryResults(ro: RelationObject): QueryResults { 120 | let qr = { 121 | columns: [], 122 | values: [] 123 | } as QueryResults; 124 | qr.columns = Object.keys(ro[0]); 125 | ro.forEach((array) => { 126 | let values: string[] = []; 127 | qr.columns.forEach((colname: any) => { 128 | values.push(array[colname] as string); 129 | }); 130 | qr.values.push(values); 131 | }); 132 | return qr; 133 | } -------------------------------------------------------------------------------- /testEndToEnd/diel/flights-remote.diel: -------------------------------------------------------------------------------- 1 | -- sampling 200 2 | -- if we change this into a view, then this actually just a random sampling upon every interaction 3 | create table sampledFlights AS 4 | select * 5 | from flights 6 | order by random() limit 200; 7 | 8 | ---------------------------------------------------------------------------------- 9 | 10 | CREATE EVENT TABLE originSelectionEvent ( 11 | origin text 12 | ); 13 | 14 | create output currentOriginSelection AS 15 | select origin, timestep 16 | from LATEST originSelectionEvent; 17 | 18 | create event table panZoomEvent ( 19 | minDelay number, 20 | maxDelay number 21 | ); 22 | 23 | create view currentPanZoom as 24 | select minDelay, maxDelay 25 | from LATEST panZoomEvent; 26 | 27 | create output allPastSelections AS 28 | select distinct origin from originSelectionEvent order by timestep; 29 | 30 | ---------------------------------------------------------------------------------- 31 | 32 | -- note: this is a bit annoying... 33 | create output allOriginAirports AS 34 | select * from allOriginAirportsEvent; 35 | 36 | create event view allOriginAirportsEvent AS 37 | select 38 | origin, 39 | count() as c 40 | from flights 41 | group by origin 42 | order by c 43 | limit 10; 44 | 45 | ---------------------------------------------------------------------------------- 46 | 47 | create event view delayDistanceByOriginEvent AS 48 | select 49 | f.delay, 50 | f.distance 51 | from sampledFlights f 52 | join currentOriginSelection s on f.origin = s.origin; 53 | 54 | insert into __scales (component, dimension, outputName, x, y) values ('flight', 2, 'delayDistanceByOrigin', 'delay', 'distance'); 55 | create output delayDistanceByOrigin AS 56 | select * 57 | from delayDistanceByOriginEvent e 58 | join currentOriginSelection s on e.request_timestep = s.timestep; 59 | 60 | ---------------------------------------------------------------------------------- 61 | 62 | insert into __scales (component, dimension, outputName, x, y, z) values ('flight', 3, 'delayDistanceHeatOut', 'delayBin', 'distanceBin', 'count'); 63 | create output delayDistanceHeatOut AS 64 | select * from delayDistanceHeatEvent; 65 | 66 | -- heatmap implementation, capped at 2000 for distance, and -100 to 500 for delay 67 | create event view delayDistanceHeatEvent AS 68 | select 69 | max(min(round(f.delay / 10) * 10, 500), -100) as delayBin, 70 | min(round(f.distance / 100) * 100, 2000) as distanceBin, 71 | count() as count 72 | from flights f 73 | group by delayBin, distanceBin; 74 | 75 | ---------------------------------------------------------------------------------- 76 | 77 | create template flightDistributionT(sourceRelation) 78 | select 79 | case when (z.maxDelay IS NULL) 80 | then 81 | round(delay / 20) * 20 82 | else 83 | round(delay / (max(round((z.maxDelay - z.minDelay) / 10), 5))) * (max(round((z.maxDelay - z.minDelay) / 10), 5)) 84 | end as delayBin, 85 | count() as count 86 | from {sourceRelation} 87 | left outer join currentPanZoom z 88 | group by delayBin 89 | having ( 90 | delayBin <= z.maxDelay 91 | and delayBin >= z.minDelay 92 | ) OR ( 93 | z.maxDelay IS NULL 94 | ) OR ( 95 | z.minDelay IS NULL 96 | ); 97 | 98 | create view filteredFlights AS 99 | select f.* 100 | from flights f 101 | join currentOriginSelection s 102 | on f.origin = s.origin; 103 | 104 | create event view flightDistributionByAirportEvent AS 105 | use template flightDistributionT(sourceRelation='filteredFlights'); 106 | 107 | create event view flightDistributionEvent AS 108 | use template flightDistributionT(sourceRelation='flights'); 109 | 110 | insert into __scales (component, dimension, outputName, x, y, z) values ('flights', 3, 'flightDistribution', 'delayBin', 'count', 'series'); 111 | create output flightDistribution AS 112 | select 113 | delayBin, 114 | count, 115 | 'all' as series 116 | from LATEST flightDistributionEvent 117 | UNION 118 | select 119 | delayBin, 120 | count, 121 | 'airport' 122 | from LATEST flightDistributionByAirportEvent; 123 | 124 | ------- 125 | 126 | CREATE EVENT TABLE zoomScatterItx( 127 | minDelay INT, maxDelay INT, minDistance INT, maxDistance INT 128 | ); 129 | 130 | create view currentZoomScaterItx AS 131 | select * from LATEST zoomScatterItx; 132 | 133 | insert into __scales (component, dimension, outputName, x, y) values ('flights', 2, 'delayByDistance', 'delay', 'distance'); 134 | CREATE OUTPUT delayByDistance AS 135 | select * 136 | from LATEST delayByDistanceEvent; 137 | 138 | CREATE EVENT VIEW delayByDistanceEvent AS 139 | SELECT delay, distance 140 | FROM flights JOIN currentZoomScaterItx z ON 141 | (delay < z.maxDelay AND delay > z.minDelay 142 | AND distance < z.maxDistance AND distance > z.minDistance) 143 | -- or (z.maxDelay IS NULL) 144 | ORDER BY ORIGIN LIMIT 100; -------------------------------------------------------------------------------- /src/compiler/DielAstGetters.ts: -------------------------------------------------------------------------------- 1 | import { LogInternalError, DielInternalErrorType } from "../util/messages"; 2 | import { SelectionUnit, DielAst, RelationType, OriginalRelation, Relation, ExprType, ExprColumnAst, RelationNameType, ExprFunAst, SimpleColumn, BuiltInColumn, DielDataType } from "../parser/dielAstTypes"; 3 | import { DerivedRelation } from ".."; 4 | import { EventTableColumns, EventViewColumns } from "./passes/distributeQueries"; 5 | 6 | const DerivedRelationTypes = new Set([RelationType.View, RelationType.EventView, , RelationType.Output, RelationType.DerivedTable]); 7 | const OriginalRelationTypes = new Set([RelationType.Table, RelationType.EventTable, RelationType.ExistingAndImmutable]); 8 | 9 | // --------------- BEGIN CHECKERs -------------------- 10 | 11 | export function IsRelationTypeDerived(rType: RelationType) { 12 | if (DerivedRelationTypes.has(rType)) { 13 | return true; 14 | } else if (OriginalRelationTypes.has(rType)) { 15 | return false; 16 | } else { 17 | LogInternalError(`RelationType ${rType} is not defined to be derived or not`); 18 | return null; 19 | } 20 | } 21 | 22 | // --------------- BEGIN GETTERS -------------------- 23 | 24 | export function DeriveColumnsFromRelation(r: Relation): SimpleColumn[] { 25 | if (IsRelationTypeDerived(r.relationType)) { 26 | const d = r as DerivedRelation; 27 | const originalColumns = DeriveColumnsFromSelectionUnit(d.selection.compositeSelections[0].relation); 28 | const derivedColumns = r.relationType === RelationType.EventView 29 | // this casting is a bit brittle; should think more about the types 30 | ? originalColumns.concat(EventViewColumns as SimpleColumn[]) 31 | : originalColumns 32 | ; 33 | return derivedColumns; 34 | } else { 35 | const o = r as OriginalRelation; 36 | const originalColumns: SimpleColumn[] = o.columns.map(c => ({ 37 | cName: c.cName, 38 | dataType: c.dataType 39 | })); 40 | const derivedColumns = r.relationType === RelationType.EventTable 41 | ? originalColumns.concat(EventTableColumns as SimpleColumn[]) 42 | : originalColumns 43 | ; 44 | return derivedColumns; 45 | } 46 | } 47 | 48 | export function DeriveColumnsFromSelectionUnit(su: SelectionUnit): SimpleColumn[] | null { 49 | const columns: SimpleColumn[] = []; 50 | if (!su.derivedColumnSelections) return LogInternalError(`These should be defined already! Might not be visiting from topoligical order.`); 51 | for (let i = 0; i < su.derivedColumnSelections.length; i ++) { 52 | const column = su.derivedColumnSelections[i]; 53 | switch (column.expr.exprType) { 54 | case ExprType.Column: 55 | const columnExpr = column.expr as ExprColumnAst; 56 | columns.push({ 57 | cName: column.alias, 58 | dataType: columnExpr.dataType 59 | }); 60 | break; 61 | case ExprType.Func: 62 | const functionExpr = column.expr as ExprFunAst; 63 | columns.push({ 64 | cName: column.alias, 65 | dataType: functionExpr.dataType 66 | }); 67 | break; 68 | default: 69 | return LogInternalError(`${column.expr.exprType} Not handled`, DielInternalErrorType.UnionTypeNotAllHandled); 70 | } 71 | } 72 | return columns; 73 | } 74 | 75 | export function GetAllDerivedViews(ast: DielAst): DerivedRelation[] { 76 | return ast.relations.filter(r => IsRelationTypeDerived(r.relationType)) as DerivedRelation[]; 77 | } 78 | 79 | export function GetAllPrograms(ast: DielAst | DielAst) { 80 | return ast.programs; 81 | } 82 | 83 | export function GetAllStaticOriginalTables(ast: DielAst): OriginalRelation[] { 84 | return ast.relations.filter(r => r.relationType === RelationType.ExistingAndImmutable) as OriginalRelation[]; 85 | } 86 | 87 | export function GetAllDielDefinedOriginalRelations(ast: DielAst): OriginalRelation[] { 88 | return ast.relations.filter(r => (r.relationType === RelationType.EventTable) 89 | || (r.relationType === RelationType.Table) 90 | ) as OriginalRelation[]; 91 | } 92 | 93 | export function GetOriginalRelations(ast: DielAst): OriginalRelation[] { 94 | return ast.relations.filter(r => !IsRelationTypeDerived(r.relationType)) as OriginalRelation[]; 95 | } 96 | 97 | export function GetAllOutputs(ast: DielAst): DerivedRelation[] { 98 | return ast.relations.filter(r => r.relationType === RelationType.Output) as DerivedRelation[]; 99 | } 100 | 101 | export function GetRelationDef(ast: DielAst, rName: string): Relation | null { 102 | // first search in original, then serach in derived, compalin otherwise 103 | if (!rName) return LogInternalError(`[GetRelationDef]: passed in null name`); 104 | const result = ast.relations.find(r => r.rName === rName); 105 | if (result) { 106 | return result; 107 | } else { 108 | return LogInternalError(`[GetRelationDef]: Relation ${rName} not defined`); 109 | } 110 | } 111 | 112 | // --------------- BEGIN DERIVERS -------------------- 113 | 114 | 115 | 116 | // -------------- --------------- 117 | 118 | export function IsRelationEvent(ast: DielAst, rName: RelationNameType) { 119 | const r = GetRelationDef(ast, rName); 120 | return (r.relationType === RelationType.EventTable) || (r.relationType === RelationType.EventView); 121 | } -------------------------------------------------------------------------------- /tests/unitTest.ts: -------------------------------------------------------------------------------- 1 | import { getTopologicalOrder } from "../src/compiler/passes/passesHelper"; 2 | import { SingleDistribution, QueryDistributionRecursiveEval } from "../src/compiler/passes/distributeQueries"; 3 | import { DbIdType, RelationNameType, RelationType } from "../src/parser/dielAstTypes"; 4 | import { LocalDbId } from "../src/compiler/DielPhysicalExecution"; 5 | import { BgGreen, Reset } from "../src/util/messages"; 6 | import { DependencyTree, NodeDependencyAugmented } from "../src/runtime/runtimeTypes"; 7 | 8 | 9 | 10 | export function testTopologicalSort() { 11 | const depTree: DependencyTree = new Map([ 12 | ["v1", { 13 | relationName: "v1", 14 | isDynamic: false, 15 | dependsOn: ["v2"], 16 | isDependedBy: [] 17 | }], 18 | ["v2", { 19 | relationName: "v2", 20 | isDynamic: false, 21 | dependsOn: ["v3"], 22 | isDependedBy: ["v1"] 23 | }], 24 | ["v3", { 25 | relationName: "v3", 26 | isDynamic: false, 27 | dependsOn: ["v4"], 28 | isDependedBy: ["v2"] 29 | }], 30 | ["v4", { 31 | relationName: "v4", 32 | isDynamic: false, 33 | dependsOn: [], 34 | isDependedBy: ["v3"] 35 | }], 36 | ]); 37 | const sorted = getTopologicalOrder(depTree); 38 | console.log("sorted", sorted); 39 | if (sorted[0] !== "v4" || sorted[1] !== "v3" || sorted[2] !== "v2" || sorted[3] !== "v1") { 40 | throw new Error(`testTopologicalSort failed`); 41 | } 42 | } 43 | 44 | export function testDistributionLogc() { 45 | // set up 46 | const augmentedDep = new Map(); 47 | const depI1: NodeDependencyAugmented = { 48 | relationName: "i1", 49 | remoteId: LocalDbId, 50 | dependsOn: [], 51 | isDynamic: true, 52 | isDependedBy: ["v1"] 53 | }; 54 | const depI2: NodeDependencyAugmented = { 55 | relationName: "i2", 56 | remoteId: LocalDbId, 57 | isDynamic: true, 58 | dependsOn: [], 59 | isDependedBy: ["v1"] 60 | }; 61 | const depR1: NodeDependencyAugmented = { 62 | relationName: "r1", 63 | remoteId: 2, 64 | isDynamic: false, 65 | dependsOn: [], 66 | isDependedBy: ["v1"] 67 | }; 68 | const depV1: NodeDependencyAugmented = { 69 | relationName: "v1", 70 | isDynamic: false, 71 | dependsOn: ["i1", "i2", "r1"], 72 | isDependedBy: ["o1"] 73 | }; 74 | const depE1: NodeDependencyAugmented = { 75 | relationName: "e1", 76 | isDynamic: false, 77 | dependsOn: ["i1", "i2", "r1"], 78 | isDependedBy: [] 79 | }; 80 | const depO1: NodeDependencyAugmented = { 81 | relationName: "o1", 82 | isDynamic: false, 83 | dependsOn: ["e1"], 84 | isDependedBy: [] 85 | }; 86 | augmentedDep.set("i1", depI1); 87 | augmentedDep.set("i2", depI2); 88 | augmentedDep.set("r1", depR1); 89 | augmentedDep.set("v1", depV1); 90 | augmentedDep.set("e1", depE1); 91 | augmentedDep.set("o1", depO1); 92 | 93 | // as a simple hack for testing, encode the remoteId by the size of the original Id 94 | function selectRelationEvalOwner (dbIds: Set): DbIdType { 95 | return Math.max(...Array.from(dbIds)); 96 | } 97 | const distributions: SingleDistribution[] = []; 98 | const relationTypeLookup = (rName: string) => { 99 | if (rName[0] === "o") return RelationType.Output; 100 | if (rName[0] === "v") return RelationType.View; 101 | if (rName[0] === "i") return RelationType.EventTable; 102 | if (rName[0] === "e") return RelationType.EventView; 103 | return RelationType.Table; 104 | }; 105 | const scope = { 106 | augmentedDep, 107 | selectRelationEvalOwner, 108 | relationTypeLookup, 109 | outputName: "dummy" 110 | }; 111 | QueryDistributionRecursiveEval(distributions, scope, "o1"); 112 | const expected = [ 113 | { 114 | relationName: "i1", 115 | from: 1, 116 | to: 1 117 | }, 118 | { 119 | relationName: "i2", 120 | from: 1, 121 | to: 1 122 | }, 123 | { 124 | relationName: "r1", 125 | from: 2, 126 | to: 2 127 | }, 128 | { 129 | relationName: "i1", 130 | from: 1, 131 | to: 2 132 | }, 133 | { 134 | relationName: "i2", 135 | from: 1, 136 | to: 2 137 | }, 138 | { 139 | relationName: "r1", 140 | from: 2, 141 | to: 2 142 | }, 143 | { 144 | relationName: "e1", 145 | from: 2, 146 | to: 1 147 | }, 148 | { 149 | relationName: "e1", 150 | from: 1, 151 | to: 1 152 | }, 153 | { 154 | relationName: "o1", 155 | from: 1, 156 | to: 1 157 | } 158 | ]; 159 | // DO assertions 160 | if (expected.length !== distributions.length) { 161 | throw new Error(`Distribution incorrect length, expected ${JSON.stringify(expected, null, 2)}, but got ${JSON.stringify(distributions, null, 2)}`); 162 | } 163 | expected.map(e => { 164 | const f = distributions.find(d => (e.relationName === d.relationName) && (e.to === d.to) && (d.from === e.from)); 165 | if (!f) { 166 | throw new Error(`Distribution incorrect! Expected to find ${JSON.stringify(e, null, 2)}`); 167 | } 168 | }); 169 | console.log(`${BgGreen}Passed testDistributionLogc!${Reset}`); 170 | } 171 | 172 | // @LUCIE care to take a stab? 173 | 174 | export function testDistributionLogcComplex() { 175 | // TODO 176 | } 177 | 178 | export function testDependencyGraph() { 179 | // TODO 180 | } -------------------------------------------------------------------------------- /src/util/dielUtils.ts: -------------------------------------------------------------------------------- 1 | // import { QueryResults, Database } from "sql.js"; 2 | import initSqlJs from "sql.js"; 3 | import { log } from "./dielUdfs"; 4 | import { LogInfo } from "./messages"; 5 | 6 | type QueryResults = any; 7 | type Database = any; 8 | 9 | 10 | export type OutputBoundFunc = (v: any) => any; 11 | 12 | export function DeepCopy(o: T): T { 13 | return JSON.parse(JSON.stringify(o)); 14 | } 15 | 16 | export function CheckObjKeys(required: string[], obj: any) { 17 | for (let i = 0; i < required.length; i ++) { 18 | if (!(required[i] in obj)) { 19 | return false; 20 | } 21 | } 22 | return true; 23 | } 24 | 25 | export function SetSymmetricDifference(setA: Set, setB: Set): Set { 26 | let _difference = new Set(setA); 27 | for (let elem of setB) { 28 | if (_difference.has(elem)) { 29 | _difference.delete(elem); 30 | } else { 31 | _difference.add(elem); 32 | } 33 | } 34 | return _difference; 35 | } 36 | 37 | /** 38 | * find elements in A but not in B 39 | * @param setA 40 | * @param setB 41 | */ 42 | export function SetDifference(setA: Set, setB: Set): Set { 43 | let _difference = new Set(setA); 44 | for (let elem of setB) { 45 | _difference.delete(elem); 46 | } 47 | return _difference; 48 | } 49 | 50 | export function IsSuperset(superset: Set, subset: Set): boolean { 51 | for (let elem of subset) { 52 | if (!superset.has(elem)) { 53 | return false; 54 | } 55 | } 56 | return true; 57 | } 58 | 59 | export function IsSetIdentical(setA: Set, setB: Set): boolean { 60 | return SetDifference(setA, setB).size === 0; 61 | } 62 | 63 | export function SetUnion(setA: Set, setB: Set): Set { 64 | let _union = new Set(setA); 65 | for (let elem of setB) { 66 | _union.add(elem); 67 | } 68 | return _union; 69 | } 70 | 71 | export function SetIntersection(setA: Set, setB: Set): Set { 72 | let _intersection = new Set(); 73 | for (let elem of setB) { 74 | if (setA.has(elem)) { 75 | _intersection.add(elem); 76 | } 77 | } 78 | return _intersection; 79 | } 80 | 81 | export async function loadDbHelper(db: Database, file: string, tick: () => () => void) { 82 | if (db) { 83 | db.close(); 84 | } 85 | let buffer; 86 | const response = await fetch(file); 87 | const bufferRaw = await response.arrayBuffer(); 88 | buffer = new Uint8Array(bufferRaw); 89 | const SQL = await initSqlJs(); 90 | db = new SQL.Database(buffer); 91 | // db = new Database(buffer); 92 | // db.create_function("timeNow", timeNow); 93 | db.create_function("log", log); 94 | db.create_function("tick", tick()); 95 | 96 | // db.create_function("tick", this.db); 97 | LogInfo(`DIEL Loaded DB Successfully`); 98 | } 99 | 100 | // console tools 101 | // export function d(db: Database, sql: string) { 102 | // let r = db.exec(sql); 103 | // if (r.length > 0) { 104 | // r[0].values.map((v) => { 105 | // v.map((c, i) => { 106 | // if (r[0].columns[i] === "ts") { 107 | // c = new Date(c as number).toDateString(); 108 | // } 109 | // }); 110 | // }); 111 | // console.log(r[0].columns.join("\t")); 112 | // console.log(JSON.stringify(r[0].values).replace(/\],\[/g, "\n").replace("[[", "").replace("]]", "").replace(/,/g, "\t")); 113 | // } else { 114 | // console.log("NO RESULT"); 115 | // } 116 | // } 117 | // if (typeof window !== "undefined" && window) (window).d = d; 118 | // debug assertions 119 | 120 | export function assertQueryHasResult(r: QueryResults, query?: string) { 121 | if ((!r) || (!r.values)) { 122 | throw new Error(`Query ${query} has NO result`); 123 | } 124 | } 125 | 126 | // this is needed because the groupby has non deterministic orderings 127 | // and I wasn't able to write a custom reducer 128 | // I suspect this is faster anyways 129 | export function hashCompare(a: string, b: string): number { 130 | // split by - 131 | const aVals = a.split("-"); 132 | const bVals = b.split("-"); 133 | for (let i = 0; i < aVals.length; i++) { 134 | const av = aVals[i]; 135 | const table = av.split(":")[0]; 136 | const bR = bVals.filter(bv => bv.split(":")[0] === table); 137 | if (bR.length !== 1) { 138 | console.log("didn't find", table, bVals); 139 | return 0; 140 | } 141 | if (bR[0] !== av) { 142 | return 0; 143 | } 144 | } 145 | return 1; 146 | } 147 | 148 | /********************** 149 | * download support 150 | *********************/ 151 | 152 | function prettyTimeNow() { 153 | return new Date().toTimeString().substr(0, 8); 154 | } 155 | 156 | export function downloadHelper(blob: Blob, name: string, extension = "db") { 157 | if (typeof window !== "undefined" && window) { 158 | let a = document.createElement("a"); 159 | a.href = window.URL.createObjectURL(blob); 160 | a.download = `${name}_${prettyTimeNow()}.${extension}`; 161 | a.onclick = function() { 162 | setTimeout(function() { 163 | window.URL.revokeObjectURL(a.href); 164 | }, 1500); 165 | }; 166 | a.click(); 167 | } else { 168 | console.log("Not in web environment!"); 169 | } 170 | } 171 | 172 | 173 | export function downloadQueryResultAsCSV(db: Database, query: string) { 174 | let csvContent = ""; 175 | let r = db.exec(query); 176 | if (r.length && r[0].values) { 177 | csvContent += r[0].columns.join(",") + "\r\n"; 178 | r[0].values.forEach((rowArray: any) => { 179 | let row = rowArray.join(","); 180 | csvContent += row + "\r\n"; 181 | }); 182 | let b = new Blob([csvContent], {type: "text/plain;charset=UTF-8"}); 183 | downloadHelper(b, "userData.csv"); 184 | console.log("should have downloaded", csvContent); 185 | } else { 186 | console.log("NO RESULT"); 187 | } 188 | } -------------------------------------------------------------------------------- /tests/compilerTests/testViewConstraints.ts: -------------------------------------------------------------------------------- 1 | import { checkViewConstraint } from "../../src/compiler/passes/generateViewConstraints"; 2 | import { ParsePlainDielAst, ParsePlainSelectQueryAst, CompileAst } from "../../src/compiler/compiler"; 3 | import { GenerateUnitTestErrorLogger } from "../testHelper"; 4 | import { TestLogger } from "../testTypes"; 5 | 6 | /** 7 | * Answer Query array order, respectively: 8 | * not null --> unique --> check constraints 9 | * See combinedAnswer1 for example. 10 | */ 11 | 12 | const notNull1 = ` 13 | create view filtered_view as select a1, a2 from t1 where a1 > 10 14 | constrain a1 NOT NULL; 15 | `; 16 | 17 | const notNullAnswer1 = [ 18 | ` 19 | select * 20 | from (select a1, a2 from t1 where a1 > 10) 21 | where a1 IS NULL 22 | limit 30 23 | ; 24 | ` 25 | ]; 26 | 27 | const notNull2 = ` 28 | create view filtered_view as select a1, a2 from t1 where a1 > 10 29 | constrain a1 NOT NULL, a2 NOT NULL; 30 | `; 31 | 32 | const notNullAnswer2 = [ 33 | ` 34 | select * 35 | from (select a1, a2 from t1 where a1 > 10) 36 | where a1 IS NULL 37 | limit 30; 38 | `, 39 | ` 40 | select * 41 | from (select a1, a2 from t1 where a1 > 10) 42 | where a2 IS NULL 43 | limit 30; 44 | ` 45 | ]; 46 | 47 | const check1 = ` 48 | create view filtered_view as select a1, a2 from t1 where a1 < 10 49 | constrain CHECK (a1 < 5); 50 | `; 51 | 52 | const checkAnswer1 = [ 53 | ` 54 | select * 55 | from (select a1, a2 from t1 where a1 < 10) 56 | where NOT (a1 < 5) 57 | limit 30; 58 | ` 59 | ]; 60 | 61 | const check2 = ` 62 | create view filtered_view as select a1, a2 from t1 where a1 < 10 63 | constrain CHECK (a1 < 5), CHECK (a2 < 10); 64 | `; 65 | 66 | const checkAnswer2 = [ 67 | ` 68 | select * 69 | from (select a1, a2 from t1 where a1 < 10) 70 | where NOT (a1 < 5) 71 | limit 30; 72 | `, 73 | ` 74 | select * 75 | from (select a1, a2 from t1 where a1 < 10) 76 | where NOT (a2 < 10) 77 | limit 30; 78 | ` 79 | ]; 80 | 81 | const check3 = ` 82 | create view filtered_view as select a1, a2 from t1 where a1 < 10 83 | constrain CHECK (a1 < 5 and a2 < 10); 84 | `; 85 | 86 | const checkAnswer3 = [ 87 | ` 88 | select * 89 | from (select a1, a2 from t1 where a1 < 10) 90 | where NOT (a1 < 5 and a2 < 10) 91 | limit 30; 92 | ` 93 | ]; 94 | 95 | const check4 = ` 96 | create view filtered_view as select a1, a2 from t1 where a1 < 10 97 | constrain CHECK (a1 < 5 or a2 < 10); 98 | ` 99 | ; 100 | 101 | const checkAnswer4 = [ 102 | ` 103 | select * 104 | from (select a1, a2 from t1 where a1 < 10) 105 | where NOT (a1 < 5 or a2 < 10) 106 | limit 30; 107 | ` 108 | ]; 109 | 110 | const unique1 = ` 111 | create view filtered_view as select a1, a2, a3 from t1 where a1 < 10 112 | constrain UNIQUE (a1, a2); 113 | `; 114 | 115 | const uniqueAnswer1 = [ 116 | ` 117 | select a1, a2, COUNT (*) 118 | from (select a1, a2, a3 from t1 where a1 < 10) 119 | GROUP BY a1, a2 HAVING COUNT(*) > 1 120 | limit 30; 121 | ` 122 | ]; 123 | 124 | const unique2 = ` 125 | create view filtered_view as select a1, a2, a3 from t1 where a1 < 10 126 | constrain UNIQUE (a1, a2), UNIQUE (a3); 127 | `; 128 | 129 | const uniqueAnswer2 = [ 130 | ` 131 | select a1, a2, COUNT (*) 132 | from (select a1, a2, a3 from t1 where a1 < 10) 133 | GROUP BY a1, a2 HAVING COUNT(*) > 1 134 | limit 30; 135 | `, 136 | ` 137 | select a3, COUNT (*) 138 | from (select a1, a2, a3 from t1 where a1 < 10) 139 | GROUP BY a3 HAVING COUNT(*) > 1 140 | limit 30; 141 | ` 142 | ]; 143 | 144 | const combined1 = ` 145 | create view filtered_view as select a1, a2, a3 from t1 146 | constrain UNIQUE(a1), CHECK (a2 < 5), a3 NOT NULL; 147 | ` 148 | ; 149 | 150 | const combinedAnswer1 = [ 151 | ` 152 | select * 153 | from (select a1, a2, a3 from t1) 154 | where a3 IS NULL 155 | limit 30; 156 | `, 157 | ` 158 | select a1, COUNT (*) 159 | from (select a1, a2, a3 from t1) 160 | GROUP BY a1 HAVING COUNT(*) > 1 161 | limit 30; 162 | `, 163 | ` 164 | select * 165 | from (select a1, a2, a3 from t1) 166 | where NOT (a2 < 5) 167 | limit 30; 168 | ` 169 | ]; 170 | 171 | const tests = [ 172 | [combined1, combinedAnswer1], 173 | [notNull1, notNullAnswer1], 174 | [notNull2, notNullAnswer2], 175 | [check1, checkAnswer1], 176 | [check2, checkAnswer2], 177 | [check3, checkAnswer3], 178 | [check4, checkAnswer4], 179 | [unique1, uniqueAnswer1], 180 | [unique2, uniqueAnswer2] 181 | ]; 182 | 183 | export function assertCheckViewConstraintTest() { 184 | const logger = GenerateUnitTestErrorLogger("assertCheckViewConstraintTest"); 185 | for (let test of tests) { 186 | const query = test[0] as string; 187 | const answer = test[1] as string[]; 188 | let ast = ParsePlainDielAst(query); 189 | CompileAst(ast); 190 | let viewqueries = checkViewConstraint(ast); 191 | let computedQueries = Array.from(viewqueries)[0][1]; 192 | compareAST(computedQueries, answer, logger); 193 | } 194 | logger.pass(); 195 | } 196 | 197 | function compareAST(computedQueries: string[][], answerQueries: string[], logger: TestLogger) { 198 | if (answerQueries.length !== computedQueries.length) { 199 | logger.error("Array length different."); 200 | return; 201 | } 202 | for (let i in computedQueries) { 203 | let q = computedQueries[i][0]; 204 | let a = answerQueries[i]; 205 | 206 | let ast1 = ParsePlainSelectQueryAst(q); 207 | let ast2 = ParsePlainSelectQueryAst(a); 208 | 209 | let pretty1 = JSON.stringify(ast1, null, 2); 210 | let pretty2 = JSON.stringify(ast2, null, 2); 211 | 212 | if (pretty1 !== pretty2) { 213 | logger.error(`${q} is not the same as ${a}`); 214 | } 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /src/compiler/passes/syntaxSugar.ts: -------------------------------------------------------------------------------- 1 | import {ReportDielUserError, LogInternalError} from "../../util/messages"; 2 | import { ExprAst, ExprType, ExprRelationAst, FunctionType, ExprFunAst, SetOperator, DielDataType, DerivedRelation, RelationReferenceType, RelationReferenceDirect, RelationReference } from "../../../src/parser/dielAstTypes"; 3 | import { SelectionUnit, DielAst, AstType } from "../../parser/dielAstTypes"; 4 | import { WalkThroughSelectionUnits } from "../DielAstVisitors"; 5 | 6 | /** 7 | * This function traverses the places where `RelationReference` might be called 8 | * e.g., DerivedRelations, SelectionUnits in programs etc. 9 | * @param ast 10 | */ 11 | export function applyLatestToAst(ast: DielAst): void { 12 | // TODO: check 13 | WalkThroughSelectionUnits(ast, applyLatestToSelectionUnit); 14 | // // first go through the derivedrelations 15 | // const derived = GetAllDerivedViews(ast); 16 | // derived.map(d => {d.selection.compositeSelections.map(c => { 17 | // applyLatestToSelectionUnit(c.relation); 18 | // }); 19 | // }); 20 | // // also need to check programs and commands 21 | // ast.programs.forEach(c => { 22 | // if (c.astType === AstType.RelationSelection) { 23 | // (c as RelationSelection).compositeSelections.map(c => { 24 | // applyLatestToSelectionUnit(c.relation); 25 | // }); 26 | // } 27 | // }); 28 | 29 | } 30 | 31 | export function ApplyLatestToDerivedRelation(derived: DerivedRelation) { 32 | derived.selection.compositeSelections.map(s => applyLatestToSelectionUnit(s.relation)); 33 | return derived; 34 | } 35 | 36 | // IN PLACE 37 | function applyLatestToRelationReference(ref: RelationReference, selection: SelectionUnit): void { 38 | if (ref.relationReferenceType === RelationReferenceType.Direct) { 39 | const r = ref as RelationReferenceDirect; 40 | if (r.isLatest) { 41 | modifyWhereComplete(selection, r.relationName); 42 | } 43 | } 44 | return; 45 | } 46 | 47 | /** 48 | * find all the RelationReference instances in the DerivedRelation ASTs 49 | * check if they say "isLatest", turn that boolean into false, and change the subquery 50 | * 51 | * report error if there is already a subquery --- LATEST can only be used with a simple 52 | * named relation 53 | * 54 | * note this will probably be recursive 55 | * @param relation 56 | */ 57 | export function applyLatestToSelectionUnit(relation: SelectionUnit): void { 58 | if (!relation.baseRelation) return; 59 | applyLatestToRelationReference(relation.baseRelation, relation); 60 | if (!relation.joinClauses) return; 61 | for (let i = 0; i < relation.joinClauses.length; i++) { 62 | applyLatestToRelationReference(relation.joinClauses[i].relation, relation); 63 | } 64 | } 65 | 66 | /** 67 | * Modify WhereClause in relation in place, appending timestep clause for the relationName. 68 | * 69 | * In other words, append the following to the ast whereclause: 70 | * relationName.timestep = (select max(relationName) from relationName). 71 | * 72 | * @param relation 73 | * @param relationName 74 | */ 75 | function modifyWhereComplete(relation: SelectionUnit, relationName: string): void { 76 | const originalWhere = relation.whereClause; 77 | // create exprast for subquery (select max(timestep) from relation) 78 | const rhsExpr = createSubquery(relationName); 79 | const whereAST: ExprFunAst = { 80 | exprType: ExprType.Func, 81 | functionType: FunctionType.Logic, 82 | functionReference: "=", 83 | dataType: DielDataType.Boolean, 84 | args: [] 85 | }; 86 | 87 | // create exprast for relation.timestep 88 | let lhsExpr: ExprAst = { 89 | exprType: ExprType.Column, 90 | columnName: "timestep", 91 | relationName: relationName 92 | }; 93 | if (originalWhere) { 94 | // Merge into a where query 95 | lhsExpr = modifyExistingWhere(originalWhere, lhsExpr); 96 | } 97 | // set the where clause in place 98 | whereAST.args = [lhsExpr, rhsExpr]; 99 | relation.whereClause = whereAST; 100 | } 101 | 102 | /** 103 | * Modify existing whereClause and return it so that a new ExprAst can be appended 104 | */ 105 | function modifyExistingWhere(originalAST: ExprAst, lhs: ExprAst): ExprAst { 106 | let andAst: ExprAst = { 107 | exprType: ExprType.Func, 108 | functionType: FunctionType.Logic, 109 | functionReference: "and", 110 | dataType: DielDataType.Boolean, 111 | args: [originalAST, lhs] 112 | }; 113 | 114 | return andAst; 115 | } 116 | 117 | /** 118 | * Create ExprAst for the clause (select max(relationName) from relationName). 119 | */ 120 | function createSubquery(relationName: string): ExprAst { 121 | const relationAST: ExprRelationAst = { 122 | exprType: ExprType.Relation, 123 | dataType: DielDataType.Relation, 124 | selection: { 125 | astType: AstType.RelationSelection, 126 | compositeSelections: [ 127 | { 128 | op: SetOperator.NA, 129 | relation: { 130 | isDistinct: false, 131 | derivedColumnSelections: [ 132 | { 133 | alias: "timestep", 134 | expr: { 135 | exprType: ExprType.Func, 136 | functionType: FunctionType.Custom, 137 | functionReference: "max", 138 | args: [ 139 | { 140 | exprType: ExprType.Column, 141 | columnName: "timestep" 142 | } 143 | ] 144 | } 145 | } 146 | ], 147 | baseRelation: { 148 | relationReferenceType: RelationReferenceType.Direct, 149 | alias: null, 150 | isLatest: false, 151 | relationName: relationName 152 | }, 153 | } 154 | } 155 | ] 156 | } 157 | }; 158 | return relationAST; 159 | } 160 | 161 | -------------------------------------------------------------------------------- /src/compiler/passes/applyTemplate.ts: -------------------------------------------------------------------------------- 1 | import { ReportDielUserError, LogInternalWarning, LogInternalError, DielInternalErrorType } from "../../util/messages"; 2 | import { ExprAst, ExprType, ExprColumnAst, ExprFunAst, ExprRelationAst, OriginalRelation, JoinAst, RelationSelection, CompositeSelectionUnit, ColumnSelection, OrderByAst, RelationReference, AstType, DielAst, RelationReferenceType, RelationReferenceDirect, RelationReferenceSubquery } from "../../parser/dielAstTypes"; 3 | import { GetAllDielDefinedOriginalRelations, GetAllDerivedViews } from "../DielAstGetters"; 4 | import { ApplyLatestToDerivedRelation } from "./syntaxSugar"; 5 | 6 | /** 7 | * Find all the top level selections for: 8 | * - views 9 | * - outputs 10 | * 11 | * Possibly also joins for 12 | * - crossfilters 13 | * 14 | * Do the copy pass for: 15 | * - inputs 16 | * - dynamicTables 17 | * @param ast diel ast 18 | */ 19 | export function ApplyTemplates(ast: DielAst) { 20 | // note: i think the concat should be fine with modifying in place? 21 | GetAllDerivedViews(ast).map(r => { 22 | TryToApplyTemplate(r.selection); 23 | // also apply syntax sugar.. 24 | ApplyLatestToDerivedRelation(r); 25 | }); 26 | 27 | // defined here since it needs to access the global definition 28 | // and the copy pass 29 | GetAllDielDefinedOriginalRelations(ast).map(r => TryToCopyRelationSpec(ast, r)); 30 | } 31 | 32 | export function TryToCopyRelationSpec(ast: DielAst, r: OriginalRelation): void { 33 | if (r.copyFrom) { 34 | // make sure it's not copying from itself 35 | if (r.copyFrom === r.rName) { 36 | ReportDielUserError(`You cannot copy ${r.rName} from itself!`); 37 | } 38 | // find the relation 39 | const sourceRelation = GetAllDielDefinedOriginalRelations(ast).filter(r => r.rName === r.copyFrom); 40 | if (sourceRelation.length === 0) { 41 | ReportDielUserError(`The relation definition you are trying to copy from, ${r.copyFrom}, does not exist`); 42 | } else { 43 | r.columns = sourceRelation[0].columns; 44 | } 45 | } 46 | } 47 | 48 | /** 49 | * modify in place 50 | * @param ast 51 | */ 52 | export function TryToApplyTemplate(ast: RelationSelection | JoinAst): void { 53 | 54 | if (!ast.templateSpec) { 55 | // there is no template to apply here! 56 | // LogInternalError(`Template variables not specified`); 57 | return; 58 | } 59 | 60 | if (ast.astType === AstType.RelationSelection) { 61 | _visitSelection(ast as RelationSelection); 62 | } else { 63 | _visitJoinAst(ast as JoinAst); 64 | } 65 | 66 | function _changeString(inStr: string | undefined): string | undefined { 67 | if (!inStr) { 68 | return inStr; 69 | } 70 | if ((inStr[0] === "{") && (inStr[inStr.length - 1] === "}")) { 71 | const varName = inStr.slice(1, inStr.length - 1); 72 | const varValue = ast.templateSpec!.get(varName); 73 | if (varValue) return varValue; 74 | LogInternalError(`${varName} was not found`); 75 | return undefined; 76 | } else { 77 | return inStr; 78 | } 79 | } 80 | 81 | function _visitColumnSelection(c: ColumnSelection) { 82 | // only change if it's a column 83 | // in theory this also needs to be recursive... but only deal with shallow stuff for now 84 | _visitExprAst(c.expr); 85 | } 86 | 87 | function _visitOrderByAst(c: OrderByAst) { 88 | _visitExprAst(c.selection); 89 | } 90 | 91 | function _visitExprAst(e: ExprAst) { 92 | if (e) { 93 | if (e.exprType === ExprType.Column) { 94 | const c = e as ExprColumnAst; 95 | const newCName = _changeString(c.columnName); 96 | if (newCName) c.columnName = newCName; 97 | const newRName = _changeString(c.relationName); 98 | if (newRName) c.relationName = newRName; 99 | } else if (e.exprType === ExprType.Func) { 100 | const f = e as ExprFunAst; 101 | // recursive! 102 | f.args.map(a => _visitExprAst(a)); 103 | } else if (e.exprType === ExprType.Relation) { 104 | const r = e as ExprRelationAst; 105 | // recursive!! 106 | _visitSelection(r.selection); 107 | } 108 | } else { 109 | LogInternalError(`Visiting null`, DielInternalErrorType.ArgNull); 110 | } 111 | } 112 | 113 | function _visitRelationReference(ref: RelationReference): void { 114 | switch (ref.relationReferenceType) { 115 | case RelationReferenceType.Direct: { 116 | const r = ref as RelationReferenceDirect; 117 | r.relationName = _changeString(r.relationName); 118 | return; 119 | } 120 | case RelationReferenceType.Subquery: { 121 | const r = ref as RelationReferenceSubquery; 122 | if (r.subquery) _visitSelection(r.subquery); 123 | return; 124 | } 125 | default: 126 | LogInternalError(``); 127 | return; 128 | } 129 | } 130 | 131 | function _visitJoinAst(j: JoinAst): void { 132 | // no nested templates allowed 133 | if (j.templateSpec) { 134 | ReportDielUserError(`No nested templates allowed`); 135 | } 136 | _visitRelationReference(j.relation); 137 | if (j.predicate) _visitExprAst(j.predicate); 138 | } 139 | 140 | function _visitCompositeSelectionUnit(ast: CompositeSelectionUnit): void { 141 | ast.relation.columnSelections.map(c => _visitColumnSelection(c)); 142 | if (ast.relation.baseRelation) _visitRelationReference(ast.relation.baseRelation); 143 | if (ast.relation.joinClauses) ast.relation.joinClauses.map(j => _visitJoinAst(j)); 144 | if (ast.relation.whereClause) _visitExprAst(ast.relation.whereClause); 145 | if (ast.relation.groupByClause) { 146 | ast.relation.groupByClause.selections.map(c => _visitExprAst(c)); 147 | if (ast.relation.groupByClause.predicate) { 148 | _visitExprAst(ast.relation.groupByClause.predicate); 149 | } 150 | } 151 | if (ast.relation.orderByClause) ast.relation.orderByClause.map(c => _visitOrderByAst(c)); 152 | if (ast.relation.limitClause) _visitExprAst(ast.relation.limitClause); 153 | } 154 | 155 | function _visitSelection(subAst: RelationSelection) { 156 | if (subAst) { 157 | subAst.compositeSelections.map(s => _visitCompositeSelectionUnit(s)); 158 | } else { 159 | LogInternalWarning(`Attempted to call __visitSelection on null`); 160 | } 161 | } 162 | } -------------------------------------------------------------------------------- /src/runtime/ConnectionWrapper.ts: -------------------------------------------------------------------------------- 1 | import { DbType, DielRemoteAction, DielRemoteReply, DielRemoteMessageId } from "./runtimeTypes"; 2 | import { ParseSqlJsWorkerResult } from "./runtimeHelper"; 3 | import { downloadHelper } from "../util/dielUtils"; 4 | import { LogInfo, LogInternalError, LogExecutionTrace, LogInternalWarning } from "../util/messages"; 5 | import { DbIdType, LogicalTimestep } from "../parser/dielAstTypes"; 6 | 7 | type FinalMsgType = 8 | { buffer: any } // setting up worker (note that talking to servers do not require serialization) 9 | | { message: string } // setting up server via socket 10 | | { sql: string } // worker 11 | | { sql: string, message: {dbName: string }} // server via socket 12 | ; 13 | 14 | // Things that need to be passed back! 15 | type FinalIdType = { 16 | remoteAction: DielRemoteAction, 17 | destinationDbId?: DbIdType, 18 | requestTimestep: LogicalTimestep 19 | }; 20 | 21 | interface FinalPromiseIdType extends FinalIdType { 22 | msgId: number; 23 | } 24 | const DielRemoteActionToEngineActionWorker = new Map([ 25 | [DielRemoteAction.GetResultsByPromise, "exec"], 26 | [DielRemoteAction.Close, "close"], 27 | [DielRemoteAction.UpdateRelation, "exec"], 28 | [DielRemoteAction.ConnectToDb, "open"], 29 | [DielRemoteAction.DefineRelations, "exec"], 30 | [DielRemoteAction.ShipRelation, "exec"], 31 | [DielRemoteAction.CleanUpQueries, "cleanup"] // supported only for socket 32 | ]); 33 | const DielRemoteActionToEngineActionSocket = new Map(DielRemoteActionToEngineActionWorker); 34 | DielRemoteActionToEngineActionSocket.set(DielRemoteAction.DefineRelations, "run"); 35 | DielRemoteActionToEngineActionSocket.set(DielRemoteAction.UpdateRelation, "run"); 36 | 37 | 38 | // to unify worker and websocket 39 | export class ConnectionWrapper { 40 | resolves: Map; 41 | rejects: Map; 42 | globalMsgId: number; 43 | remoteType: DbType; 44 | connection: Worker | WebSocket; 45 | 46 | constructor(connection: Worker | WebSocket, remoteType: DbType) { 47 | this.remoteType = remoteType; 48 | this.connection = connection; 49 | this.globalMsgId = 1; 50 | this.resolves = new Map(); 51 | this.rejects = new Map(); 52 | } 53 | 54 | public send(id: FinalIdType, msgToSend: FinalMsgType, isPromise: boolean): Promise | null { 55 | LogExecutionTrace(`Executing to DB ${(msgToSend as any).sql}`); // REMOVE 56 | if (isPromise) { 57 | // do the promise thing here 58 | const msgId = this.globalMsgId; 59 | this.globalMsgId += 1; 60 | const self = this; 61 | return new Promise(function (resolve, reject) { 62 | // save callbacks for later 63 | self.resolves.set(msgId, resolve); 64 | self.rejects.set(msgId, reject); 65 | self.sendNoPromise({...id, msgId}, msgToSend); 66 | }); 67 | } else { 68 | this.sendNoPromise(id, msgToSend); 69 | return null; 70 | } 71 | } 72 | 73 | private sendNoPromise(id: FinalIdType | FinalPromiseIdType, msgToSend: FinalMsgType) { 74 | if (this.remoteType === DbType.Worker) { 75 | // FIXME: might need some adaptor logic here to make worker the same as socket 76 | const action = DielRemoteActionToEngineActionWorker.get(id.remoteAction); 77 | if (!action) { 78 | LogInternalError(`Action must be defined, but not for ${id.remoteAction}`); 79 | } 80 | const worker = (this.connection as Worker); 81 | // FIXME: JSON.stringify(id) ? 82 | const finalMsg = { 83 | id, 84 | action, 85 | ...msgToSend 86 | }; 87 | LogExecutionTrace(`Posting to Worker`, finalMsg); 88 | worker.postMessage(finalMsg); 89 | } else { 90 | // FIXME: clear serialization logic 91 | const action = DielRemoteActionToEngineActionSocket.get(id.remoteAction); 92 | const newMessage = { 93 | id, 94 | action, 95 | ...msgToSend 96 | }; 97 | (this.connection as WebSocket).send(JSON.stringify(newMessage)); 98 | } 99 | } 100 | 101 | // FIXME: maybe don't need this since they have the same name 102 | setHandler(f: (msg: DielRemoteReply) => void) { 103 | const self = this; 104 | const newF = (event: any) => { 105 | LogExecutionTrace(`DB executed message`, event.data); 106 | let msg: DielRemoteReply | undefined; 107 | if (this.remoteType === DbType.Socket) { 108 | try { 109 | msg = parseDielReply(event.data); 110 | // special debugging case 111 | if ((msg as any).id === "test") return; 112 | } catch (e) { 113 | LogInternalWarning(`Socket sent mal-formatted message: ${JSON.stringify(event.data, null, 2)}`); 114 | return; 115 | } 116 | } else { 117 | if (event.data.id) { 118 | // special debugging case 119 | if (event.data.id === "test") return; 120 | if (event.data.id === "download") { 121 | let blob = new Blob([event.data.buffer]); 122 | downloadHelper(blob, "workerSession"); 123 | return; 124 | } 125 | msg = { 126 | id: event.data.id as DielRemoteMessageId, 127 | results: ParseSqlJsWorkerResult(event.data.results), 128 | err: event.data.err 129 | }; 130 | } else { 131 | console.log(`%c\nWorker sent mal-formatted message: ${event.data}`, "color: red"); 132 | return; 133 | } 134 | } 135 | if (msg.id.msgId) { 136 | const promiseId = msg.id.msgId; 137 | if (msg.err) { 138 | // error condition 139 | const reject = self.rejects.get(promiseId); 140 | if (reject) { 141 | reject(msg.err); 142 | } 143 | } else { 144 | const resolve = self.resolves.get(promiseId); 145 | if (resolve) { 146 | resolve(msg); 147 | } 148 | } 149 | // LogExecutionTrace("Promise resolved", 1, msg); 150 | // purge used callbacks 151 | self.resolves.delete(promiseId); 152 | self.rejects.delete(promiseId); 153 | } 154 | f(msg); 155 | }; 156 | if (this.remoteType === DbType.Worker) { 157 | (this.connection as Worker).onmessage = newF; 158 | } else { 159 | (this.connection as WebSocket).onmessage = newF; 160 | } 161 | } 162 | } 163 | 164 | function parseDielReply(rawStr: string): DielRemoteReply { 165 | let msg = JSON.parse(rawStr); 166 | // msg.id = JSON.parse(msg.id); 167 | return msg; 168 | } -------------------------------------------------------------------------------- /src/compiler/passes/inferType.ts: -------------------------------------------------------------------------------- 1 | import { LogInternalError, ReportDielUserError, DielInternalErrorType, LogInternalWarning } from "../../util/messages"; 2 | import { DielDataType, BuiltInColumnTyppes, SelectionUnit, RelationReference, ExprType, ExprFunAst, ExprColumnAst, ExprAst, BuiltInFunc, ExprValAst, ExprParen, DerivedRelation, DielAst, RelationReferenceType, RelationReferenceSubquery, Relation, OriginalRelation, CompositeSelection, RelationReferenceDirect } from "../../parser/dielAstTypes"; 3 | import { GetRelationDef, IsRelationTypeDerived } from "../DielAstGetters"; 4 | import { WalkThroughSelectionUnits } from "../DielAstVisitors"; 5 | 6 | export function InferType(ast: DielAst) { 7 | WalkThroughSelectionUnits(ast, ((selection) => { 8 | _inferTypeForSelectionUnit(ast, selection); 9 | })); 10 | } 11 | 12 | export function InferTypeForDerivedRelation(ast: DielAst, view: DerivedRelation) { 13 | view.selection.compositeSelections.map(s => { 14 | _inferTypeForSelectionUnit(ast, s.relation); 15 | }); 16 | } 17 | 18 | function _inferTypeForSelectionUnit(ast: DielAst, selection: SelectionUnit) { 19 | if (!selection.derivedColumnSelections) { 20 | LogInternalError(`Normalization pass of column selection should be defined before infer type is called! for derivedColumnSelections `); 21 | } 22 | 23 | selection.derivedColumnSelections.map(cs => { 24 | if (!cs.expr.dataType) { 25 | cs.expr.dataType = getTypeForExpr(ast, cs.expr, selection); 26 | } 27 | }); 28 | } 29 | 30 | function getUdfType(ast: DielAst, sUnit: SelectionUnit, funName: string, expr: ExprFunAst): DielDataType | null { 31 | const normalizedName = funName.toLocaleUpperCase(); 32 | switch (normalizedName) { 33 | case BuiltInFunc.Coalesce: 34 | case BuiltInFunc.IfThisThen: 35 | // 0 is the case clause, which is a boolean 36 | const firstExpr = expr.args[1]; 37 | if (!firstExpr) { 38 | LogInternalError(`If else then should contain the if clause, but is missing: ${JSON.stringify(expr, null, 2)}`); 39 | } 40 | return getTypeForExpr(ast, firstExpr, sUnit); 41 | default: 42 | const r = ast.udfTypes.find(u => u.udf === normalizedName); 43 | if (!r) { 44 | return ReportDielUserError(`Type of ${funName} not defined. Original query is ${JSON.stringify(sUnit, null, 2)}.`); 45 | } 46 | return r.type; 47 | } 48 | } 49 | 50 | // recursive! 51 | function getTypeForExpr(ast: DielAst, expr: ExprAst, sUnit: SelectionUnit): DielDataType | null { 52 | switch (expr.exprType) { 53 | case ExprType.Func: 54 | const funExpr = expr as ExprFunAst; 55 | return getUdfType(ast, sUnit, funExpr.functionReference, funExpr); 56 | case ExprType.Column: 57 | const columnExpr = expr as ExprColumnAst; 58 | const cn = columnExpr.columnName; 59 | // case 1: check for keywords 60 | const special = BuiltInColumnTyppes.filter(sc => sc.column === cn)[0]; 61 | if (special) { 62 | return special.type; 63 | } 64 | // make sure that the source is specified 65 | if (!columnExpr.relationName) { 66 | return LogInternalError(`The normalization pass screwed up and did not specify the source relation for ${JSON.stringify(columnExpr)}`); 67 | } 68 | // case 2: see if its from the baseRelation 69 | if (sUnit.baseRelation) { 70 | if (columnExpr.relationName === sUnit.baseRelation.alias) { 71 | return getColumnTypeFromReference(ast, columnExpr.columnName, sUnit.baseRelation); 72 | } 73 | // case 3: see if its from the joins 74 | if (sUnit.joinClauses) { 75 | for (let i = 0; i < sUnit.joinClauses.length; i ++) { 76 | const j = sUnit.joinClauses[i]; 77 | if (columnExpr.relationName === j.relation.alias) { 78 | return getColumnTypeFromReference(ast, columnExpr.columnName, j.relation); 79 | } 80 | } 81 | } 82 | } 83 | // FIXME: should pass some metadata for debugging 84 | debugger; 85 | ReportDielUserError(`type not found for column`); 86 | case ExprType.Parenthesis: 87 | const parenExpr = expr as ExprParen; 88 | // unpack 89 | return getTypeForExpr(ast, parenExpr.content, sUnit); 90 | case ExprType.Val: 91 | const valExpr = expr as ExprValAst; 92 | return valExpr.dataType; 93 | case ExprType.Star: 94 | return LogInternalError(`Should only invoke or normalized selections`); 95 | default: 96 | return LogInternalError(`Should have handled all cases`, DielInternalErrorType.UnionTypeNotAllHandled); 97 | } 98 | } 99 | 100 | function getColumnTypeFromCompositionSelection(s: CompositeSelection, columnName: string): DielDataType | null { 101 | const selections = s[0].relation.derivedColumnSelections; 102 | if (!selections) return LogInternalError(`Should have done the normalization pass before this for column ${columnName} in composite selction:\n ${JSON.stringify(s, null, 2)}!\n`); 103 | for (let i = 0; i < selections.length; i ++) { 104 | const s = selections[i]; 105 | if (columnName === s.alias) return s.expr.dataType; 106 | } 107 | LogInternalWarning(`Type not found for ${columnName}`); 108 | return null; 109 | } 110 | 111 | // the two types are interchangeable for access purposes. 112 | // exported for tests! 113 | export function getColumnTypeFromRelation(relation: Relation, columnName: string): DielDataType | null { 114 | // case 1: derived 115 | if (IsRelationTypeDerived(relation.relationType)) { 116 | return getColumnTypeFromCompositionSelection((relation as DerivedRelation).selection.compositeSelections, columnName); 117 | } 118 | // case 2: original 119 | const column = (relation as OriginalRelation).columns.filter(r => r.cName === columnName); 120 | if (column.length > 0) { 121 | return column[0].dataType; 122 | } else { 123 | LogInternalWarning(`Type not found for ${columnName}`); 124 | return null; 125 | } 126 | } 127 | 128 | function getColumnTypeFromReference(ast: DielAst, columnName: string, ref: RelationReference): DielDataType | null { 129 | switch (ref.relationReferenceType) { 130 | case RelationReferenceType.Direct: { 131 | const r = ref as RelationReferenceDirect; 132 | // we need the original name, not alias! 133 | const rDef = GetRelationDef(ast, r.relationName); 134 | return getColumnTypeFromRelation(rDef, columnName); 135 | } 136 | case RelationReferenceType.Subquery: { 137 | const rDef = (ref as RelationReferenceSubquery).subquery.compositeSelections; 138 | return getColumnTypeFromCompositionSelection(rDef, columnName); 139 | } 140 | default: 141 | return LogInternalError(``); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/compiler/passes/materialization.ts: -------------------------------------------------------------------------------- 1 | import { getTopologicalOrder } from "./passesHelper"; 2 | import { DeriveOriginalRelationsAViewDependsOn, DeriveDepTreeFromSqlRelations } from "./dependency"; 3 | import { DependencyTree } from "../../runtime/runtimeTypes"; 4 | import { GetColumnsFromSelection } from "./distributeQueries"; 5 | import { SqlAst, SqlRelationType, SqlDerivedRelation, SqlRelation } from "../../parser/sqlAstTypes"; 6 | import { LogInternalError } from "../../util/messages"; 7 | import { Command, DeleteClause, AstType, InsertionClause, RelationSelection, RelationNameType, DerivedRelation, OriginalRelation, RelationConstraints } from "../../parser/dielAstTypes"; 8 | 9 | /** 10 | * For now we wil just set the changes on all original tables 11 | * @param ast 12 | */ 13 | export function TransformAstForMaterialization(ast: SqlAst) { 14 | const dynamic = new Set(ast.relations.filter(r => r.isDynamic).map(r => r.rName)); 15 | const views = ast.relations.filter(r => r.relationType === SqlRelationType.View) as SqlDerivedRelation[]; 16 | const deps = DeriveDepTreeFromSqlRelations(views, dynamic); 17 | 18 | // get topo order 19 | const topoOrder = getTopologicalOrder(deps); 20 | 21 | function getRelationDef(rName: string) { 22 | return ast.relations.find(v => v.rName === rName); 23 | } 24 | const toMaterialize = getRelationsToMateralize(deps, getRelationDef); 25 | 26 | let originalTables: Set; 27 | // Materialize by topological order 28 | topoOrder.forEach(relation => { 29 | if (toMaterialize.indexOf(relation) !== -1) { 30 | const view = getRelationDef(relation); 31 | if (!view) { 32 | LogInternalError(`${relation} was not found!`); 33 | } else { 34 | originalTables = DeriveOriginalRelationsAViewDependsOn(deps, view.rName); 35 | if (originalTables.size === 0) { 36 | // this means that this is a static view, in that it just need to be populated once... 37 | // we then just need to change its reference to a table 38 | // set in place 39 | view.relationType = SqlRelationType.Table; 40 | } else { 41 | materializeAView(view as SqlDerivedRelation, ast, originalTables); 42 | } 43 | } 44 | } 45 | }); 46 | } 47 | 48 | /** 49 | * Change the derived view ast into program ast in place 50 | */ 51 | function materializeAView(view: SqlDerivedRelation, ast: SqlAst, originalTables: Set) { 52 | const columns = GetColumnsFromSelection(view.selection); 53 | if (!columns) { 54 | LogInternalError(`Columsn for ${view.selection} undefined`); 55 | return; 56 | } 57 | let table = { 58 | relationType: SqlRelationType.Table, 59 | rName: view.rName, 60 | columns 61 | }; 62 | 63 | // 2. make a program ast 64 | // 2-1. create insert,delete ast 65 | let deleteCommand = makeDeleteCommand(view.rName); 66 | let insertCommand = makeInsertCommand(view); 67 | 68 | // 3. push into programs. (this is supposed to preserve topo order) 69 | originalTables.forEach(rName => { 70 | // if the tname already exists in the map, append the program 71 | const existingTrigger = ast.triggers.find(t => t.afterRelationName === rName); 72 | if (existingTrigger) { 73 | existingTrigger.commands.push(deleteCommand, insertCommand); 74 | } else { 75 | ast.triggers.push({ 76 | tName: `${rName}Trigger`, 77 | afterRelationName: rName, 78 | commands: [deleteCommand, insertCommand] 79 | }); 80 | } 81 | }); 82 | 83 | // 4. build the final ast. The order of relations sometimes changes 84 | // since table -> view -> output order. 85 | let relationIndex = ast.relations.indexOf(view); 86 | ast.relations[relationIndex] = table; 87 | } 88 | 89 | /** 90 | * Translate view constraints to table constraints. 91 | * @param view 92 | * @param table 93 | */ 94 | function translateConstraints(view: DerivedRelation, table: OriginalRelation) { 95 | if (view.constraints) { 96 | // 1. translate column constraints 97 | table.columns.forEach(c => { 98 | // 1-1. Handle NOT NULL constraint 99 | if (view.constraints.notNull.indexOf(c.cName) !== -1) { 100 | c.constraints.notNull = true; 101 | } 102 | // 1-2. Handle UNIQUE column constraint 103 | view.constraints.uniques.forEach(array => { 104 | if (array.length === 1 && array[0] === c.cName) { 105 | c.constraints.unique = true; 106 | } 107 | }); 108 | // 1-3. No need to translate check constraints 109 | // they are directly copied in step 2, at the end. 110 | }); 111 | // 2. copy relation constraints 112 | table.constraints = view.constraints; 113 | } else { 114 | table.constraints = { 115 | relationNotNull: false, 116 | relationHasOneRow: false, 117 | primaryKey: [], 118 | notNull: [], 119 | uniques: [], 120 | exprChecks: [], 121 | foreignKeys: [], 122 | } as RelationConstraints; 123 | } 124 | } 125 | 126 | 127 | /** 128 | * Create AST for DeleteClause 129 | * e.g) delete from v2; 130 | * @param view 131 | */ 132 | function makeDeleteCommand(viewName: string): Command { 133 | let deleteClause: DeleteClause; 134 | deleteClause = { 135 | astType: AstType.Delete, 136 | relationName: viewName, 137 | predicate: undefined 138 | }; 139 | return deleteClause; 140 | } 141 | 142 | /** 143 | * Create AST for InsertClause 144 | * e.g) insert into v2 select a + 1 as aPrime from v1; 145 | * @param view 146 | */ 147 | function makeInsertCommand(view: SqlDerivedRelation): Command { 148 | let insertClause: InsertionClause; 149 | insertClause = { 150 | astType: AstType.Insert, 151 | relation: view.rName, 152 | columns: [], 153 | selection: { 154 | astType: AstType.RelationSelection, 155 | compositeSelections: view.selection 156 | } as RelationSelection 157 | }; 158 | return insertClause; 159 | } 160 | 161 | /** 162 | * take in dependency tree and a relation definition lookup function 163 | * o1 164 | * / 165 | * t1 -> v1 - o2 166 | * @param ast 167 | */ 168 | function getRelationsToMateralize(depTree: DependencyTree, getRelationDef: (rName: RelationNameType) => SqlRelation | undefined ): string[] { 169 | let toMAterialize: RelationNameType[] = []; 170 | depTree.forEach((nodeDep, relationName) => { 171 | const rDef = getRelationDef(relationName); 172 | if (rDef && (rDef.relationType === SqlRelationType.View)) { 173 | // and if the view is dependent on by at least two views/outputs, mark it as to materialize 174 | if (nodeDep.isDependedBy.length > 1) { 175 | toMAterialize.push(relationName); 176 | } 177 | } 178 | }); 179 | return toMAterialize; 180 | } -------------------------------------------------------------------------------- /src/runtime/asyncPolicies.ts: -------------------------------------------------------------------------------- 1 | import { RelationType, SetOperator, JoinAst, ExprAst, ExprFunAst, FunctionType, AstType, ExprType, DielDataType, JoinType, CompositeSelectionUnit, RelationSelection, DerivedRelation, SelectionUnit, ExprColumnAst, RelationReferenceType, BuiltInColumn, RelationNameType } from "../parser/dielAstTypes"; 2 | import { ApplyLatestToDerivedRelation } from "../compiler/passes/syntaxSugar"; 3 | import { LogInternalError } from "../util/messages"; 4 | 5 | function getTimestepColumn(relationName: string, request = false): ExprColumnAst { 6 | return { 7 | exprType: ExprType.Column, 8 | dataType: DielDataType.Number, 9 | columnName: request ? BuiltInColumn.REQUEST_TIMESTEP : BuiltInColumn.TIMESTEP, 10 | relationName 11 | }; 12 | } 13 | 14 | function defaultPolicyGetOutputHelper(ast: DerivedRelation, asyncViewName: string, eventDeps: Set): DerivedRelation { 15 | // const derivedColumnSelections = ast.selection.compositeSelections[0].relation.derivedColumnSelections; 16 | // const compositeSelections = ast.selection.compositeSelections.map((s) => { 17 | // // LAZY 18 | // const relation = JSON.parse(JSON.stringify(s.relation)) as SelectionUnit; 19 | // // we just need to take the alias (assume normalized), and add the evenViewName 20 | 21 | const originalDerivedColumnSelections = ast.selection.compositeSelections[0].relation.derivedColumnSelections; 22 | if (!originalDerivedColumnSelections) { 23 | LogInternalError(`AST for ${ast.rName} should be compiled`); 24 | } 25 | const derivedColumnSelections = originalDerivedColumnSelections.map(c => { 26 | if (!c.alias) LogInternalError(`Alias should be normalized already!`); 27 | const newExpr: ExprColumnAst = { 28 | exprType: ExprType.Column, 29 | columnName: c.alias, 30 | relationName: asyncViewName, 31 | dataType: c.expr.dataType, 32 | }; 33 | return { 34 | expr: newExpr, 35 | alias: c.alias 36 | }; 37 | }); 38 | // if (eventDeps.size === 0) { 39 | const newOutput: DerivedRelation = { 40 | rName: ast.rName, 41 | relationType: RelationType.Output, 42 | selection: { 43 | astType: AstType.RelationSelection, 44 | compositeSelections: [{ 45 | op: SetOperator.NA, 46 | relation: { 47 | isDistinct: false, 48 | derivedColumnSelections, 49 | baseRelation: { 50 | relationReferenceType: RelationReferenceType.Direct, 51 | isLatest: true, 52 | relationName: asyncViewName 53 | } 54 | } 55 | }] 56 | } 57 | }; 58 | return ApplyLatestToDerivedRelation(newOutput); 59 | // FIXME: the following logic is a bit bogus... 60 | // } else { 61 | // let timeStepSelections: SelectionUnit[] = []; 62 | // eventDeps.forEach(e => { 63 | // timeStepSelections.push({ 64 | // isDistinct: false, 65 | // derivedColumnSelections: [{ 66 | // expr: getTimestepColumn(e), 67 | // alias: BuiltInColumn.TIMESTEP, 68 | // }], 69 | // baseRelation: { 70 | // relationReferenceType: RelationReferenceType.Direct, 71 | // relationName: e, 72 | // } 73 | // }); 74 | // }); 75 | // const unionedTimestepRelationName = "timestepUnion"; 76 | // const compositeSelections = timeStepSelections.map((relation, i) => { 77 | // if (i) { 78 | // return { 79 | // op: SetOperator.UNION, 80 | // relation 81 | // }; 82 | // } else { 83 | // return { 84 | // op: SetOperator.NA, 85 | // relation 86 | // }; 87 | // } 88 | // }); 89 | // const unionTimestepRelation: RelationSelection = { 90 | // astType: AstType.RelationSelection, 91 | // compositeSelections 92 | // }; 93 | // const maxTimeStepRelationName = "maxTimeStep"; 94 | // const maxColumnSelection: RelationSelection = { 95 | // astType: AstType.RelationSelection, 96 | // compositeSelections: [{ 97 | // op: SetOperator.NA, 98 | // relation: { 99 | // derivedColumnSelections: [{ 100 | // expr: { 101 | // exprType: ExprType.Column, 102 | // dataType: DielDataType.Number, 103 | // functionType: FunctionType.Math, 104 | // functionReference: "max", 105 | // args: [getTimestepColumn(unionedTimestepRelationName)] 106 | // }, 107 | // alias: BuiltInColumn.TIMESTEP 108 | // }], 109 | // baseRelation: { 110 | // relationReferenceType: RelationReferenceType.Subquery, 111 | // subquery: unionTimestepRelation, 112 | // alias: unionedTimestepRelationName 113 | // } 114 | // } 115 | // }] 116 | // }; 117 | // let joinClauses: JoinAst[] = [{ 118 | // astType: AstType.Join, 119 | // joinType: JoinType.Inner, 120 | // relation: { 121 | // relationReferenceType: RelationReferenceType.Subquery, 122 | // subquery: maxColumnSelection, 123 | // alias: maxTimeStepRelationName 124 | // }, 125 | // predicate: { 126 | // exprType: ExprType.Func, 127 | // dataType: DielDataType.Boolean, 128 | // functionType: FunctionType.Compare, 129 | // functionReference: "=", 130 | // args: [getTimestepColumn(asyncViewName, true), getTimestepColumn(maxTimeStepRelationName)] 131 | // } 132 | // }]; 133 | // const newOutput: DerivedRelation = { 134 | // rName: ast.rName, 135 | // relationType: RelationType.Output, 136 | // selection: { 137 | // astType: AstType.RelationSelection, 138 | // compositeSelections: [{ 139 | // op: SetOperator.NA, 140 | // relation: { 141 | // derivedColumnSelections, 142 | // baseRelation: { 143 | // relationReferenceType: RelationReferenceType.Direct, 144 | // relationName: asyncViewName 145 | // }, 146 | // joinClauses 147 | // } 148 | // }] 149 | // } 150 | // }; 151 | // return newOutput; 152 | // } 153 | } 154 | 155 | export function GetAsyncViewNameFromOutput(rName: RelationNameType) { 156 | return `${rName}AsyncView`; 157 | } 158 | 159 | /** 160 | * Note that we are passing the input dependencies here because 161 | * otherwise it needs to get access to the IR/depTree 162 | * an example would be 163 | SELECT * 164 | FROM distDataEvent e 165 | JOIN LATEST slideItx i ON i.timestep = e.request_timestep; 166 | 167 | where request_timestep = (select max timestep from slideItx union ) 168 | we need to find the latest of all the inputs, if there are more than one 169 | * @param ast 170 | * @param inputDeps 171 | */ 172 | export function OutputToAsyncDefaultPolicty(ast: DerivedRelation, eventDeps: Set) { 173 | console.log("applying default async", ast.rName); 174 | // first create a new async view from the output 175 | // again, copy by reference, should be a bit careful here 176 | // note that we cannot _just_ copy over, the relation name would have changed. 177 | const viewName = GetAsyncViewNameFromOutput(ast.rName); 178 | 179 | const asyncView: DerivedRelation = { 180 | rName: viewName, 181 | relationType: RelationType.EventView, 182 | selection: ast.selection 183 | }; 184 | const output = defaultPolicyGetOutputHelper(ast, viewName, eventDeps); 185 | return { 186 | asyncView, 187 | output 188 | }; 189 | } -------------------------------------------------------------------------------- /tests/compilerTests/testConstraintMaterializedView.ts: -------------------------------------------------------------------------------- 1 | // import { GenerateUnitTestErrorLogger } from "../testHelper"; 2 | // import { TransformAstForMaterialization } from "../../src/compiler/passes/materialization"; 3 | // import { ParsePlainDielAst } from "../../build/src/compiler/compiler"; 4 | 5 | // // @LUCIE: seems like this is WIP? 6 | 7 | // // Relation vs column constraint 8 | // // create view v1 ,,,, constrain check (a < 10); 9 | // // create view v1 ,,,, constrain unique (a); 10 | // // create view v1 ,,,, constrain a not null; 11 | 12 | 13 | // // NOT NULL constraint 14 | // // 1) column constraint 15 | // // 1-1. single column constraint. materialize v1. 16 | // let notNull1 = 17 | // ` 18 | // create event table t1 (a integer); 19 | 20 | // create view v1 as select a + 1 as aPrime, a + 2 as aPPrime from t1 21 | // constrain aPrime NOT NULL, aPPrime NOT NULL; 22 | 23 | // create output o1 as select aPrime from v1; 24 | // create output o2 as select aPrime from v1; 25 | // `; 26 | 27 | // let notNullAnswer1 = 28 | // ` 29 | // create event table t1 (a integer); 30 | 31 | // create table v1 ( 32 | // aPrime integer NOT NULL, 33 | // aPPrime integer NOT NULL 34 | // ); 35 | // create program after (t1) 36 | // begin 37 | // delete from v1; 38 | // insert into v1 select a + 1 as aPrime, a + 2 as aPPrime from t1; 39 | // end; 40 | 41 | // create output o1 as select aPrime from v1; 42 | // create output o2 as select aPrime from v1; 43 | // ` 44 | // ; 45 | 46 | 47 | // // CHECK constraint 48 | // // 1) column constraint 49 | // let check1 = 50 | // ` 51 | // create event table t1 (a integer); 52 | 53 | // create view v1 as select a + 1 as aPrime, a + 2 as aPPrime from t1 54 | // constrain check (aPrime > 10 and aPrime < 100), check (aPPrime < 100); 55 | 56 | // create output o1 as select aPrime from v1; 57 | // create output o2 as select aPrime from v1; 58 | // `; 59 | 60 | // let checkAnswer1 = 61 | // ` 62 | // create event table t1 (a integer); 63 | 64 | // create table v1 ( 65 | // aPrime integer, 66 | // aPPrime integer, 67 | // check (aPrime > 10 and aPrime < 100), 68 | // check (aPPrime < 100) 69 | // ); 70 | // create program after (t1) 71 | // begin 72 | // delete from v1; 73 | // insert into v1 select a + 1 as aPrime, a + 2 as aPPrime from t1; 74 | // end; 75 | 76 | // create output o1 as select aPrime from v1; 77 | // create output o2 as select aPrime from v1; 78 | // ` 79 | // ; 80 | 81 | // // 2) relation constraint 82 | // let check2 = 83 | // ` 84 | // create event table t1 (a integer); 85 | 86 | // create view v1 as select a + 1 as aPrime, a + 2 as aPPrime from t1 87 | // constrain check (aPrime > 10 and aPPrime < 100), check (aPrime < 15); 88 | 89 | // create output o1 as select aPrime from v1; 90 | // create output o2 as select aPrime from v1; 91 | // `; 92 | 93 | // let checkAnswer2 = 94 | // ` 95 | // create event table t1 (a integer); 96 | 97 | // create table v1 ( 98 | // aPrime integer, 99 | // aPPrime integer, 100 | // check (aPrime > 10 and aPPrime < 100), 101 | // check (aPrime < 15) 102 | // ); 103 | // create program after (t1) 104 | // begin 105 | // delete from v1; 106 | // insert into v1 select a + 1 as aPrime, a + 2 as aPPrime from t1; 107 | // end; 108 | 109 | // create output o1 as select aPrime from v1; 110 | // create output o2 as select aPrime from v1; 111 | // ` 112 | // ; 113 | 114 | 115 | // // UNIQUE constraint 116 | // // 1) column constraint 117 | // let unique1 = 118 | // ` 119 | // create event table t1 (a integer); 120 | 121 | // create view v1 as select a + 1 as aPrime, a + 2 as aPPrime from t1 122 | // constrain UNIQUE (aPrime), UNIQUE(aPPrime); 123 | 124 | // create output o1 as select aPrime from v1; 125 | // create output o2 as select aPrime from v1; 126 | // `; 127 | 128 | // let uniqueAnswer1 = 129 | // ` 130 | // create event table t1 (a integer); 131 | 132 | // create table v1 ( 133 | // aPrime integer UNIQUE, 134 | // aPPrime integer UNIQUE 135 | // ); 136 | // create program after (t1) 137 | // begin 138 | // delete from v1; 139 | // insert into v1 select a + 1 as aPrime, a + 2 as aPPrime from t1; 140 | // end; 141 | 142 | // create output o1 as select aPrime from v1; 143 | // create output o2 as select aPrime from v1; 144 | // ` 145 | // ; 146 | 147 | // // 2) relation constraint 148 | // let unique2 = 149 | // ` 150 | // create event table t1 (a integer); 151 | 152 | // create view v1 as select a + 1 as aPrime, a + 2 as aPPrime from t1 153 | // constrain UNIQUE (aPrime, aPPrime), UNIQUE(aPrime); 154 | 155 | // create output o1 as select aPrime from v1; 156 | // create output o2 as select aPrime from v1; 157 | // `; 158 | 159 | // let uniqueAnswer2 = 160 | // ` 161 | // create event table t1 (a integer); 162 | 163 | // create table v1 ( 164 | // aPrime integer UNIQUE, 165 | // aPPrime integer, 166 | // unique (aPrime, aPPrime) 167 | // ); 168 | // create program after (t1) 169 | // begin 170 | // delete from v1; 171 | // insert into v1 select a + 1 as aPrime, a + 2 as aPPrime from t1; 172 | // end; 173 | 174 | // create output o1 as select aPrime from v1; 175 | // create output o2 as select aPrime from v1; 176 | // ` 177 | // ; 178 | 179 | 180 | // let combined1 = 181 | // ` 182 | // create event table t1 (a integer); 183 | 184 | // create view v1 as select a + 1 as aPrime, a + 2 as aPPrime from t1 185 | // constrain 186 | // aPrime NOT NULL, 187 | // check (aPrime > 10), 188 | // aPPrime NOT NULL, 189 | // check (aPPrime > 40 and aPPrime < 100), 190 | // unique (aPrime, aPPrime), 191 | // unique (aPrime), 192 | // unique (aPPrime), 193 | // check (aPrime > 10 and aPPrime < 100); 194 | 195 | // create output o1 as select aPrime from v1; 196 | // create output o2 as select aPrime from v1; 197 | // `; 198 | 199 | // let combinedAnswer1 = 200 | // ` 201 | // create event table t1 (a integer); 202 | 203 | // create table v1 ( 204 | // aPrime integer UNIQUE NOT NULL, 205 | // aPPrime integer UNIQUE NOT NULL, 206 | // check (aPrime > 10), 207 | // unique (aPrime, aPPrime), 208 | // check (aPPrime > 40 and aPPrime < 100), 209 | // check (aPrime > 10 and aPPrime < 100) 210 | // ); 211 | // create program after (t1) 212 | // begin 213 | // delete from v1; 214 | // insert into v1 select a + 1 as aPrime, a + 2 as aPPrime from t1; 215 | // end; 216 | 217 | // create output o1 as select aPrime from v1; 218 | // create output o2 as select aPrime from v1; 219 | // `; 220 | 221 | // const tests = [ 222 | // [combined1, combinedAnswer1], 223 | // [notNull1, notNullAnswer1], 224 | // [check1, checkAnswer1], 225 | // [check2, checkAnswer2], 226 | // [unique1, uniqueAnswer1], 227 | // [unique2, uniqueAnswer2] 228 | // ]; 229 | 230 | // export function testMaterializedViewConstraint() { 231 | // const logger = GenerateUnitTestErrorLogger("testMaterializedViewConstraint"); 232 | // for (let test of tests) { 233 | // let query = test[0]; 234 | // let answer = test[1]; 235 | // const ast1 = ParsePlainDielAst(query); 236 | 237 | // let ast2 = ParsePlainDielAst(answer); 238 | // // materialize the test query 239 | // TransformAstForMaterialization(ast1); 240 | 241 | // let pretty1 = JSON.stringify(ast1, null, 2); 242 | // let pretty2 = JSON.stringify(ast2, null, 2); 243 | // // compare ast except for program 244 | // if (pretty1 !== pretty2) { 245 | // logger.error(`${pretty1} is not the same as ${pretty2}`); 246 | // } 247 | // // compare program 248 | // let map1 = JSON.stringify(Array.from(ast1.programs)); 249 | // let map2 = JSON.stringify(Array.from(ast2.programs)); 250 | // if (map1 !== map2) { 251 | // logger.error(`${map1} is not the same as ${map2}`); 252 | // } 253 | // } 254 | // logger.pass(); 255 | // } -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # DIEL 2 | 3 | Welcome to the DIEL home page. 4 | 5 | ## The Current State 6 | 7 | Scaling a data analysis app from client-only to client-server requires a lot of engineering changes---setting up the APIs on the server, optimizing data processing (e.g., caching, prefetching), and coordinating networked requests with user interaction. 8 | 9 | This doesn't have to be the case. Here's how. 10 | 11 | ## The UI (State) as a Query 12 | 13 | When you query a database on the cloud, you do not have to specify _how_ to access the data, but merely _declare_ the logic. We think you can do the same with data that is on the client and data that is on the server. 14 | 15 | The critical step is to take on a (_very_) data-centric lens, where we see interactions as data, too [1](#meijer). This observation applies to events in general. By unifying events with data, we can now define the state of the UI as a query. 16 | 17 | The query can reference data that live on remote servers and the full event history, and the query execution engine will figure out how to keep the UI state up to date with the new values in the tables, which is to say, imperatively, that an event automatically causes the UI to reevaluate. 18 | 19 | ## Hello, DIEL 20 | 21 | You can view the [live examples](https://logical-interactions.github.io/diel-gallery/). 22 | 23 | ### Counter Example 24 | 25 | Let's start with the simplest example to outline the flow of coding in DIEL. Let's consider a counter that increments and decrements by clicks on respective buttons. 26 | 27 | In DIEL's model, we can store the click events as a row in a table, which we can call `clickEvent`. To differentiate between increments and decrements, we define the `delta` column, for each click on increment, the delta is `1`, and decrement is `-1`. To get the current count, we can sum up the `deltas. 28 | 29 | ![counter steps](https://i.ibb.co/8DBm4JJ/counter-steps.png) 30 | 31 | Below illustrates the set up using a plain database on the client ([sql.js](https://github.com/kripken/sql.js/))[2](#clientdb), without any custom diel logic. 32 | 33 | ![counter flow](https://i.ibb.co/4ZRt2nj/counter-flow.png) 34 | 35 | ### Chart Example 36 | 37 | The previous example is indeed a _counter_ example because while it works, it makes no sense why anyone would load a giant js library (sql.js) and write in a different language (SQL), to do something trivial in a few lines of JavaScript code---but hopefully, you see how it works. 38 | 39 | Now we look at a better-motivated example---interactive visualization. The setting here is that we have two tables of information. One is that of registration for a conference and for each participant their arrival and departure times, along with information of what animal they like (the example was originally for a talk at ForwardJS with the intention to produce humor). 40 | 41 | First, let's see how we can use SQL to derive the static charts---the first scatter plot is just a raw selection, and the bar chart is a distribution which we can compute with a simple `group by` in SQL----`select count(), animal from attendee group by animal`. 42 | 43 | To make the chart dynamic is to say that the chart _adapts_ to the user specifications, which, in DIEL, is to say that the data for the chart is derived from both the data tables and the interaction tables, via relational _join_s. This idea is illustrated in the query on the lower right. The join with `brushEvent` performs the hit testing---checking to see if the attendee's information falls into the range. The join with register "links" the logic between the two charts---the filter is on attendee, which is identified by their emails. 44 | 45 | ![chart example with code](https://i.ibb.co/R07zQtH/chart-example.png) 46 | 47 | As we have seen in the previous example, we need to reactively evaluate the views to keep the UI up to date---this is where the DIEL engine comes in. The DIEL engine triggers an event loop whenever there is a new event and reevaluates all the views that have dependencies on the event that has been invoked[3](#remote). The overall flow is shown in the image below. On the bottom are "original" relations, tables store raw data, and the above are the queries that derive new relations based on the base data and event tables, and finally on the top layer we have the _output_ views whose results are consumed by _rendering function_s that take in the tabular data and generates the DOM elements (you can use e.g., D3, Vega-Lite, or vanilla JS). 48 | 49 | ![chart model local](https://i.ibb.co/z2NGYV1/model.png) 50 | 51 | ### Real-time Example 52 | 53 | Let's take the previous example a further---what happens if the `registry` table now is being updated in real time? In DIEL, we will change the registry table from a normal table, which is static, to an event---`CREATE EVENT TABLE registry (arrival INT, departure INT, email TEXT)`. And this change will keep the brush selection up to date with the streamed registry data. Below is an example illustrating how the bar chart dependent on the new streamed in data in the scatter plot will be updated immediately. 54 | 55 | ![streaming example](https://i.ibb.co/SmLsFZP/streaming-example.png) 56 | 57 | And below is the flow of the streaming example. In contrast to the previous one, the only thing that has changed is that the `registry` table is now an event table. 58 | 59 | ![streaming model](https://i.ibb.co/5Mmb7q3/streaming-flow.png) 60 | 61 | ## Using DIEL 62 | 63 | Interested in trying out DIEL? You can install DIEL with `npm install diel`, or link [the js file](https://www.jsdelivr.com/package/npm/diel) directly in your HTML. 64 | 65 | We are working on adding documentation and a full tutorial! Meanwhile, you can check out the [gallery](https://logical-interactions.github.io/diel-gallery/). 66 | 67 | ## The Vision 68 | 69 | DIEL help us make use of many database techniques in **distributed query execution** and **materialized view maintainance**---our current [prototype](https://github.com/logical-interactions/diel) does not even scratch the surface. DIEL is both completely stateful and stateless at the same time---it's stateful because all the causes for UI changes are recorded, and stateless because there is no intermediate state that is manipulated. For a more extensive (and academic) discussion, you can read [our submitted paper on DIEL](https://www.dropbox.com/s/777bah44ca7x2lu/diel_infovis_2019.pdf?dl=0); 70 | 71 | Our hope is that by breaking the client-server architecture and making data more accessible, there can be more flexible **end-user programming** that are not confined to the limited APIs provided. And by restricting the data structure to relations and programming to relational operators, we hope to make **generating custom interactions** easier. Imagine using a tool where for every analysis task you can quickly assemble together an interactive interface to play with the data, instead of having to manually invoke functions with different parameters, or worse copy-paste cells with some changed configurations. 72 | 73 | ## People 74 | 75 | Research @ UC Berkley: [yifan](http://twitter.com/yifanwu), advised by [eugene wu](http://www.cs.columbia.edu/~ewu/), [remco chang](http://www.cs.tufts.edu/~remco/), and [joe hellerstein](http://db.cs.berkeley.edu/jmh/), with contributions from [ryan](http://github.com/rmpurp) and [lucie](http://github.com/dkqntiqn). 76 | 77 | 1: [Your Mouse is a Database 78 | ](https://queue.acm.org/detail.cfm?id=2169076) by Erik Meijer 79 | 80 | 2: Many people have the misconception of databases being a monstrous piece of software that's managed by DBAs. However, at its simplest, a database engine simply stores tables and executes relational queries, which are very simple---select, project, and join (with group by and aggregations). 81 | 82 | 3: This description omits the details of how DIEL executes across nodes---we'll update this document soon with details of how that works! 83 | -------------------------------------------------------------------------------- /tests/compilerTests/testMaterialization.ts: -------------------------------------------------------------------------------- 1 | // import { getDielIr, GetPlainDielAst } from "../../src/compiler/compiler"; 2 | // import { DielAst } from "../../src/parser/dielAstTypes"; 3 | // import { TransformAstForMaterialization } from "../../src/compiler/passes/materialization"; 4 | // import { GenerateUnitTestErrorLogger } from "../testHelper"; 5 | // import { TestLogger } from "../testTypes"; 6 | 7 | // export function testMaterialization() { 8 | // const logger = GenerateUnitTestErrorLogger("assertBasicMaterialization"); 9 | // for (let test of tests) { 10 | // let query = test[0]; 11 | // let answer = test[1]; 12 | // let ast = getDielIr(query).ast; 13 | // TransformAstForMaterialization(ast); 14 | // compareAST(answer, ast, logger); 15 | // } 16 | // logger.pass(); 17 | // } 18 | 19 | 20 | // function compareAST(query: string, ast2: DielAst, logger: TestLogger) { 21 | // let ir1 = getDielIr(query); 22 | // let ast1 = ir1.ast; 23 | 24 | // let pretty1 = JSON.stringify(ast1, null, 2); 25 | // let pretty2 = JSON.stringify(ast2, null, 2); 26 | 27 | // let map1 = JSON.stringify(Array.from(ast1.programs)); 28 | // let map2 = JSON.stringify(Array.from(ast2.programs)); 29 | 30 | // if (map1 !== map2) { 31 | // logger.error(`${map1} is not the same as ${map2}.`); 32 | // } 33 | 34 | // if (pretty1 !== pretty2) { 35 | // logger.error(`${pretty1} is not the same as ${pretty2}.`); 36 | // } 37 | // } 38 | 39 | // // 1. simple. Materialize v1 40 | // let q1 = 41 | // ` 42 | // create event table t1 (a integer); 43 | // create event table t2 (a integer); 44 | // create event table t3 (a integer); 45 | 46 | // create view v1 as select a + 1 as aPrime from t1 where a > 2; 47 | 48 | // create output o1 as select aPrime from v1 join t2 on aPrime = a; 49 | // create output o2 as select aPrime from v1 join t3 on aPrime = a; 50 | // `; 51 | 52 | // let a1 = 53 | // ` 54 | // create event table t1 (a integer); 55 | // create event table t2 (a integer); 56 | // create event table t3 (a integer); 57 | 58 | // create table v1 (aPrime integer); 59 | // create program after (t1) 60 | // begin 61 | // delete from v1; 62 | // insert into v1 select a + 1 as aPrime from t1 where a > 2; 63 | // end; 64 | 65 | // create output o1 as select aPrime from v1 join t2 on aPrime = a; 66 | // create output o2 as select aPrime from v1 join t3 on aPrime = a; 67 | // `; 68 | 69 | // // 2. multiple views to materialize horizontally. Materialize v1, v2 70 | // let q2 = 71 | // ` 72 | // create event table t1 (a integer); 73 | // create event table t2 (a integer); 74 | 75 | // create view v1 as select a + 1 as aPrime from t1 where a > 2; 76 | // create view v2 as select a + 1 as aPrime from t2 where a > 2; 77 | 78 | // create output o1 as select aPrime from v1 join v2 on aPrime = a; 79 | // create output o2 as select aPrime from v2 join v1 on aPrime = a; 80 | // `; 81 | 82 | 83 | // let a2 = 84 | // ` 85 | // create event table t1 (a integer); 86 | // create event table t2 (a integer); 87 | 88 | // create table v1 (aPrime integer); 89 | // create program after (t1) 90 | // begin 91 | // delete from v1; 92 | // insert into v1 select a + 1 as aPrime from t1 where a > 2; 93 | // end; 94 | 95 | // create table v2 (aPrime integer); 96 | // create program after (t2) 97 | // begin 98 | // delete from v2; 99 | // insert into v2 select a + 1 as aPrime from t2 where a > 2; 100 | // end; 101 | 102 | // create output o1 as select aPrime from v1 join v2 on aPrime = a; 103 | // create output o2 as select aPrime from v2 join v1 on aPrime = a; 104 | // ` 105 | // ; 106 | 107 | // // 3. only views that have more than 1 dependency. Materialize v2 108 | // let q3 = 109 | // ` 110 | // create event table t1 (a integer); 111 | // create event table t2 (a integer); 112 | // create event table t3 (a integer); 113 | 114 | // create view v1 as select a + 1 as aPrime from t1 where a > 2; 115 | // create view v2 as select a + 1 as aPrime from t2 where a > 2; 116 | // create view v3 as select a + 1 as aPrime from t3 where a > 2; 117 | 118 | // create output o1 as select aPrime from v1 join v2 on aPrime = a; 119 | // create output o2 as select aPrime from v2 join v3 on aPrime = a; 120 | // `; 121 | 122 | // let a3 = 123 | // ` 124 | // create event table t1 (a integer); 125 | // create event table t2 (a integer); 126 | // create event table t3 (a integer); 127 | 128 | // create view v1 as select a + 1 as aPrime from t1 where a > 2; 129 | 130 | // create table v2 (aPrime integer); 131 | // create program after (t2) 132 | // begin 133 | // delete from v2; 134 | // insert into v2 select a + 1 as aPrime from t2 where a > 2; 135 | // end; 136 | 137 | // create view v3 as select a + 1 as aPrime from t3 where a > 2; 138 | 139 | // create output o1 as select aPrime from v1 join v2 on aPrime = a; 140 | // create output o2 as select aPrime from v2 join v3 on aPrime = a; 141 | // `; 142 | 143 | // // 4. nested views. Materialize just v2. v2 is still dependent on v1. 144 | // let q4 = 145 | // ` 146 | // create event table t1 (a integer); 147 | 148 | // create view v1 as select a + 1 as aPrime from t1 where a > 2; 149 | // create view v2 as select a + 1 as aPrime from v1 where a > 2; 150 | 151 | // create output o1 as select aPrime from v2 where aPrime = a; 152 | // create output o2 as select aPrime from v2 where aPrime = a; 153 | // `; 154 | 155 | // let a4 = 156 | // ` 157 | // create event table t1 (a integer); 158 | 159 | // create view v1 as select a + 1 as aPrime from t1 where a > 2; 160 | 161 | // create table v2 (aPrime integer); 162 | // create program after (t1) 163 | // begin 164 | // delete from v2; 165 | // insert into v2 select a + 1 as aPrime from v1 where a > 2; 166 | // end; 167 | 168 | // create output o1 as select aPrime from v2 where aPrime = a; 169 | // create output o2 as select aPrime from v2 where aPrime = a; 170 | // `; 171 | 172 | 173 | // // 5. complex view query with Latest 174 | // let q5 = ` 175 | // create event table t1 (a integer); 176 | 177 | // create view v1 as 178 | // select a + 1 as aPrime, a+2 as aPPrime from LATEST t1 179 | // where aPrime > 10 180 | // group by aPPrime 181 | // order by count DESC 182 | // limit 10; 183 | 184 | // create output o1 as select aPrime from v1 where aPrime = a; 185 | // create output o2 as select aPrime from v1 where aPrime = a; 186 | 187 | // `; 188 | 189 | // let a5 = ` 190 | // create event table t1 (a integer); 191 | 192 | // create table v1 (aPrime integer, aPPrime integer); 193 | // create program after (t1) 194 | // begin 195 | // delete from v1; 196 | // insert into v1 197 | // select a + 1 as aPrime, a+2 as aPPrime from LATEST t1 198 | // where aPrime > 10 199 | // group by aPPrime 200 | // order by count DESC 201 | // limit 10; 202 | // end; 203 | 204 | // create output o1 as select aPrime from v1 where aPrime = a; 205 | // create output o2 as select aPrime from v1 where aPrime = a; 206 | 207 | // `; 208 | 209 | // // 6. view join query 210 | // let q6 = 211 | // ` 212 | // create event table t1 (a integer); 213 | // create event table t2 (a integer); 214 | 215 | // create view v1 as 216 | // select a + 1 as aPrime from t1 join t2 on t1.a = t2.a; 217 | 218 | // create output o1 as select aPrime from v1 where aPrime = a; 219 | // create output o2 as select aPrime from v1 where aPrime = a; 220 | 221 | // `; 222 | 223 | // let a6 = ` 224 | 225 | // create event table t1 (a integer); 226 | // create event table t2 (a integer); 227 | 228 | // create table v1 (aPrime integer); 229 | // create program after (t1, t2) 230 | // begin 231 | // delete from v1; 232 | // insert into v1 233 | // select a + 1 as aPrime from t1 join t2 on t1.a = t2.a; 234 | 235 | // end; 236 | 237 | // create output o1 as select aPrime from v1 where aPrime = a; 238 | // create output o2 as select aPrime from v1 where aPrime = a; 239 | 240 | // `; 241 | 242 | 243 | // /** 244 | // * 7. Deep Tree 245 | // Materialize v1, v2 246 | // t1 247 | // | 248 | // v1 249 | // / \ 250 | // o1 v2 251 | // / \ 252 | // o2 o3 253 | // */ 254 | // let q7 = 255 | // ` 256 | // create event table t1 (a integer); 257 | 258 | // create view v1 as select a + 1 as v1Prime from t1; 259 | // create view v2 as select v1Prime + 1 as v2Prime from v1; 260 | 261 | // create output o1 as select v1Prime from v1; 262 | // create output o2 as select v2Prime from v2; 263 | // create output o3 as select v2Prime from v2; 264 | // ` 265 | // ; 266 | 267 | // let a7 = 268 | // ` 269 | // create event table t1 (a integer); 270 | 271 | // create table v1 (v1Prime integer); 272 | // create program after (t1) 273 | // begin 274 | // delete from v1; 275 | // insert into v1 select a + 1 as v1Prime from t1; 276 | // end; 277 | 278 | // create table v2 (v2Prime integer); 279 | // create program after (t1) 280 | // begin 281 | // delete from v2; 282 | // insert into v2 select v1Prime + 1 as v2Prime from v1; 283 | // end; 284 | 285 | // create output o1 as select v1Prime from v1; 286 | // create output o2 as select v2Prime from v2; 287 | // create output o3 as select v2Prime from v2; 288 | // ` 289 | // ; 290 | 291 | // let tests = [[q1, a1], [q2, a2], [q3, a3], [q4, a4], [q5, a5], [q6, a6], [q7, a7]]; 292 | -------------------------------------------------------------------------------- /src/compiler/passes/generateViewConstraints.ts: -------------------------------------------------------------------------------- 1 | import { DielDataType, RelationType, DerivedRelation, CompositeSelection, SelectionUnit, ColumnSelection, RelationSelection, RelationReference, GroupByAst, ExprStarAst } from "../../parser/dielAstTypes"; 2 | import {GetSqlStringFromExpr, SqlStrFromSelectionUnit} from "../../compiler/codegen/codeGenSql"; 3 | import { DielAst, RelationConstraints, ExprAst, ExprParen, ExprColumnAst, ExprValAst, ExprType, FunctionType, BuiltInFunc, ExprFunAst } from "../../parser/dielAstTypes"; 4 | import { ANTLRInputStream, CommonTokenStream } from "antlr4ts"; 5 | import * as lexer from "../../parser/grammar/DIELLexer"; 6 | import * as parser from "../../parser/grammar/DIELParser"; 7 | import Visitor from "../../parser/generateAst"; 8 | import { GetAllDerivedViews } from "../DielAstGetters"; 9 | 10 | export function generateViewConstraintCheckQuery(query: string): Map { 11 | let ast = checkValidView(query); 12 | if (ast) { 13 | // valid view 14 | return checkViewConstraint(ast); 15 | } 16 | return null; 17 | } 18 | 19 | export function viewConstraintCheck(ast: DielAst): Map { 20 | return checkViewConstraint(ast); 21 | } 22 | 23 | // @LUCIE: this is already defined as a helper function in the compiler.ts file 24 | // Check if this is a valid view query. Return ast if it is, or null. 25 | function checkValidView(query: string): DielAst { 26 | const inputStream = new ANTLRInputStream(query); 27 | const l = new lexer.DIELLexer(inputStream); 28 | const tokenStream = new CommonTokenStream(l); 29 | const p = new parser.DIELParser(tokenStream); 30 | const tree = p.queries(); 31 | let visitor = new Visitor(); 32 | let ast = visitor.visitQueries(tree); 33 | if (ast.relations.length > 0) { 34 | return ast; 35 | } 36 | return null; 37 | } 38 | 39 | // Precondition: query is a valid view statement 40 | // supports only a single relation in view 41 | export function checkViewConstraint(ast: DielAst): Map { 42 | // var i, j; 43 | const ret = new Map(); 44 | 45 | // Handling multiple view statements 46 | const views = GetAllDerivedViews(ast); 47 | 48 | for (let view of views) { 49 | let viewConstraint = view.constraints; 50 | let queries = [] as string[][]; 51 | let selClause; 52 | 53 | // Only when there is a constraint on view 54 | if (viewConstraint) { 55 | let compositeSelections = view.selection.compositeSelections as CompositeSelection; 56 | 57 | // 1. handle null constraint 58 | selClause = getSelectClauseAST(compositeSelections); 59 | let nullQueries = getNullQuery(viewConstraint, selClause); 60 | queries = queries.concat(nullQueries); 61 | 62 | // 2. handle unique constraint 63 | selClause = getSelectClauseAST(compositeSelections); 64 | let uniqueQueries = getUniqueQuery(viewConstraint, selClause); 65 | queries = queries.concat(uniqueQueries); 66 | 67 | // 3. handle check constraint 68 | selClause = getSelectClauseAST(compositeSelections); 69 | let checkQueries = getCheckQuery(viewConstraint, selClause); 70 | queries = queries.concat(checkQueries); 71 | } 72 | ret.set(view.rName, queries); 73 | } 74 | return ret; 75 | } 76 | 77 | /** 78 | * Takes in a selection clause ast and copy/re-format so that 79 | * it can go inside where clause. 80 | * e.g) where (select a1, a2 from t1 where a1 < 10) 81 | */ 82 | function getSelectClauseAST(fromSel: CompositeSelection): SelectionUnit { 83 | const expr: ExprStarAst = { 84 | dataType: undefined, 85 | exprType: ExprType.Star 86 | }; 87 | const columnSel: ColumnSelection = { 88 | expr 89 | }; 90 | 91 | const baseRelation = { 92 | subquery: { 93 | compositeSelections: fromSel 94 | } as RelationSelection 95 | } as RelationReference; 96 | 97 | const selUnit: SelectionUnit = { 98 | isDistinct: undefined, 99 | columnSelections: [columnSel], 100 | baseRelation: baseRelation, 101 | derivedColumnSelections: [], 102 | joinClauses: [], 103 | whereClause: undefined, 104 | groupByClause: undefined, 105 | orderByClause: undefined, 106 | limitClause: undefined 107 | }; 108 | return selUnit; 109 | } 110 | 111 | function getCheckQuery(viewConstraint: RelationConstraints, selUnit: SelectionUnit): string[][] { 112 | let exprAsts = viewConstraint.exprChecks; 113 | let ret = [] as string[][]; 114 | let whichConstraint: string; 115 | if (exprAsts && exprAsts.length > 0) { 116 | let i, exprAst, whereClause; 117 | // iterate over check constraints 118 | for (i = 0; i < exprAsts.length; i++) { 119 | // ast for where clause 120 | exprAst = exprAsts[i] as ExprParen; 121 | 122 | // where in the query it was broken 123 | whichConstraint = "CHECK " + GetSqlStringFromExpr(exprAst); 124 | whereClause = { 125 | exprType: ExprType.Func, 126 | dataType: DielDataType.Boolean, 127 | functionType: FunctionType.Custom, 128 | functionReference: "NOT", 129 | args: [exprAst.content] // strip out parenthesis 130 | } as ExprFunAst; 131 | 132 | // ast for the whole clause 133 | selUnit.whereClause = whereClause; 134 | let str = SqlStrFromSelectionUnit(selUnit); 135 | ret.push([str, whichConstraint]); 136 | } 137 | } 138 | return ret; 139 | } 140 | 141 | 142 | function getNullQuery(viewConstraint: RelationConstraints, selUnit: SelectionUnit): string[][] { 143 | // console.log(JSON.stringify(view_constraint, null, 2)); 144 | let ret = [] as string[][]; 145 | if (viewConstraint.notNull && viewConstraint.notNull.length > 0) { 146 | 147 | let notNullColumns = viewConstraint.notNull; 148 | for (let i = 0; i < notNullColumns.length; i++) { 149 | // formating the AST for whereclause 150 | let whereClause = { 151 | exprType : ExprType.Func, 152 | dataType : DielDataType.Boolean, 153 | functionType : FunctionType.BuiltIn, 154 | functionReference : BuiltInFunc.ValueIsNull, 155 | args : [] as ExprValAst[] 156 | } as ExprFunAst; 157 | 158 | // format argument AST 159 | let cname = notNullColumns[i]; 160 | 161 | 162 | let whichConstraint = cname + " NOT NULL"; 163 | 164 | const whereClauseArg: ExprColumnAst = { 165 | exprType: ExprType.Column, 166 | columnName: cname 167 | }; 168 | 169 | whereClause.args.push(whereClauseArg); 170 | 171 | // formatting the rest of the selection unit for Not NULl 172 | selUnit.whereClause = whereClause; 173 | 174 | // Generate proper query from AST 175 | let str = SqlStrFromSelectionUnit(selUnit); 176 | ret.push([str, whichConstraint]); 177 | } 178 | } 179 | return ret; 180 | } 181 | 182 | function getUniqueQuery (viewConstraints: RelationConstraints, selUnit: SelectionUnit): string[][] { 183 | let ret = [] as string[][]; 184 | let uniques = viewConstraints.uniques; 185 | 186 | // check if unique constraint exists 187 | if (uniques && uniques.length > 0) { 188 | let str, i, groupbyArgs, groupByClause, predicateSel; 189 | 190 | for (i = 0; i < uniques.length; i++) { 191 | groupbyArgs = uniques[i]; 192 | // which constraint it was broken 193 | let whichConstraint = `UNIQUE (${groupbyArgs})`; 194 | let groupbySelections = [] as ExprColumnAst[]; 195 | 196 | // format groupby AST 197 | // which coloum to group by 198 | groupbyArgs.map(function(colName) { 199 | let expr = { 200 | exprType: ExprType.Column, 201 | dataType: undefined, 202 | hasStar: false, 203 | columnName: colName 204 | } as ExprColumnAst; 205 | groupbySelections.push(expr); 206 | }); 207 | 208 | 209 | // format having clause AST 210 | predicateSel = { 211 | exprType: ExprType.Func, 212 | functionType: FunctionType.Logic, 213 | functionReference: ">", 214 | dataType: DielDataType.Boolean, 215 | args : [ 216 | { 217 | exprType: ExprType.Func, 218 | dataType: DielDataType.Number, 219 | functionType: FunctionType.Custom, 220 | functionReference: "COUNT", 221 | args: [{ 222 | exprType: ExprType.Column, 223 | hasStar: true 224 | }] 225 | }, 226 | { 227 | exprType: ExprType.Val, 228 | dataType: DielDataType.Number, 229 | value: 1 230 | } as ExprValAst 231 | ] as ExprAst[] 232 | } as ExprFunAst; 233 | 234 | // format the whole groupby clause AST 235 | groupByClause = { 236 | selections: groupbySelections, 237 | predicate: predicateSel 238 | } as GroupByAst; 239 | 240 | 241 | selUnit.groupByClause = groupByClause; 242 | 243 | // change selUnit select clause 244 | let selectColumns = [] as ColumnSelection[]; 245 | groupbySelections.map(expr => { 246 | selectColumns.push({expr}); 247 | }); 248 | const cS: ColumnSelection = { 249 | expr: { 250 | exprType: ExprType.Func, 251 | dataType: DielDataType.Number, 252 | functionType: FunctionType.Custom, 253 | functionReference: "COUNT", 254 | args: [{ 255 | exprType: ExprType.Star 256 | }] 257 | } 258 | }; 259 | selectColumns.push(cS); 260 | 261 | selUnit.columnSelections = selectColumns as ColumnSelection[]; 262 | 263 | // Generate proper query from AST 264 | str = SqlStrFromSelectionUnit(selUnit); 265 | ret.push([str, whichConstraint]); 266 | } 267 | } 268 | return ret; 269 | } 270 | 271 | 272 | 273 | 274 | -------------------------------------------------------------------------------- /src/compiler/passes/dependency.ts: -------------------------------------------------------------------------------- 1 | import { GetAllDerivedViews, IsRelationEvent, GetRelationDef } from "../DielAstGetters"; 2 | import { DependencyTree, NodeDependency } from "../../runtime/runtimeTypes"; 3 | import { SqlDerivedRelation } from "../../parser/sqlAstTypes"; 4 | import { LogInternalError } from "../../util/messages"; 5 | import { RelationNameType, DielAst, RelationType, CompositeSelection, SelectionUnit, RelationReference, ExprAst, ExprType, ExprRelationAst, ExprFunAst, ExprParen, RelationReferenceType, RelationReferenceDirect, RelationReferenceSubquery, DerivedRelation, Relation } from "../../parser/dielAstTypes"; 6 | import { getTopologicalOrder } from "./passesHelper"; 7 | import { SetSymmetricDifference } from "../../util/dielUtils"; 8 | 9 | 10 | export function ArrangeInTopologicalOrder(ast: DielAst): void { 11 | if (!ast.depTree) return LogInternalError(`the dependency should be defined already!`); 12 | const orderedNames = getTopologicalOrder(ast.depTree); 13 | // TMP 14 | const newRelations: Relation[] = []; 15 | orderedNames.map(n => { 16 | const def = GetRelationDef(ast, n); 17 | if (def) newRelations.push(def); 18 | }); 19 | function checkDiff() { 20 | const topo = new Set(newRelations.map(n => n.rName)); 21 | const relation = new Set(ast.relations.map(r => r.rName)); 22 | const diff = SetSymmetricDifference(topo, relation); 23 | if (diff.size > 0) LogInternalError(`Topological sort length did not match, ${diff}`); 24 | } 25 | 26 | // there are cases when relations that are not read by anyone is just not inserted, i.e. no deps etc. 27 | if (newRelations.length > ast.relations.length) { 28 | checkDiff; 29 | } else if (newRelations.length === ast.relations.length) { 30 | ast.relations = newRelations; 31 | return; 32 | } else { 33 | const diff = SetSymmetricDifference(new Set(newRelations.map(n => n.rName)), new Set(ast.relations.map(r => r.rName))); 34 | diff.forEach(d => { 35 | const def = GetRelationDef(ast, d); 36 | if (def) newRelations.push(def); 37 | }); 38 | checkDiff; 39 | ast.relations = newRelations; 40 | } 41 | } 42 | /** 43 | * return the set of the relations that depent on the table passed in 44 | * TODO add depndsOn?: true and do another pass that uses transitive closure to figure out all dependnecies 45 | * @param depTree 46 | * @param rName 47 | * @param depndsOn the boolean is defaulted to true, if it's false, it's the other direction. 48 | */ 49 | export function DeriveDependentRelations(depTree: DependencyTree, rName: RelationNameType): Set { 50 | const allDependencies = new Set(); 51 | oneStep(rName, allDependencies); 52 | // recursively checks for dependencies 53 | function oneStep(rName: string, newDependencies: Set) { 54 | // search through dependency 55 | let oldSet = new Set(newDependencies); 56 | for (let [key, value] of depTree) { 57 | const found = value.dependsOn.filter(d => d === rName); 58 | if (found.length > 0) { 59 | newDependencies.add(key); 60 | } 61 | } 62 | // set difference 63 | const diff = new Set([...newDependencies].filter(x => !oldSet.has(x))); 64 | if (diff.size > 0) { 65 | // need to run this on more dependencies 66 | diff.forEach((v) => { 67 | oneStep(v, newDependencies); 68 | }); 69 | } 70 | return newDependencies; 71 | } 72 | return allDependencies; 73 | } 74 | 75 | export function DeriveDepTreeFromSqlRelations(views: SqlDerivedRelation[], dynamic: Set): DependencyTree { 76 | const depTree: DependencyTree = new Map(); 77 | const isDynamic = (rName: RelationNameType) => { 78 | return dynamic.has(rName); 79 | }; 80 | views.map(v => AddSingleDependency(depTree, v.selection, v.rName, isDynamic)); 81 | return depTree; 82 | } 83 | 84 | /** 85 | * Get the set of relations that the view depends on 86 | * needs to iterate until we hit the original tables 87 | * @param viewName 88 | * @param depTree 89 | */ 90 | export function DeriveOriginalRelationsAViewDependsOn(depTree: DependencyTree, viewName: string): Set | null { 91 | 92 | let dep = depTree.get(viewName); 93 | let tables = new Set (); 94 | if (dep && dep.dependsOn.length > 0) { 95 | // breadth first 96 | let toVisit = dep.dependsOn.slice(); 97 | let visited = [viewName] as string[]; 98 | 99 | let next: string | undefined; 100 | while (toVisit.length > 0) { 101 | next = toVisit.shift(); 102 | if (next) { 103 | const dep = depTree.get(next); 104 | if (!dep) { 105 | return LogInternalError(`Dependency ${dep} not found`); 106 | } 107 | if (dep.isDynamic) { 108 | tables.add(next); 109 | visited.push(next); 110 | if (dep.dependsOn.length !== 0) { 111 | LogInternalError(`dynamic tables should have zero dependencies!`); 112 | } 113 | continue; 114 | } else { 115 | let children = dep.dependsOn; 116 | children.forEach(child => { 117 | if (toVisit.indexOf(next) === -1 && visited.indexOf(next) === -1) { 118 | toVisit.push(child); 119 | } 120 | }); 121 | visited.push(next); 122 | } 123 | } 124 | } 125 | } 126 | return tables; 127 | } 128 | 129 | 130 | // ------------- BEGIN SETTER -------------------- 131 | export function AddDepTree(ast: DielAst): DependencyTree { 132 | const isDynamic = (rName: string) => { 133 | const found = ast.relations.find(r => r.rName === rName); 134 | return (found && found.relationType === RelationType.EventTable) ? true : false; 135 | }; 136 | GetAllDerivedViews(ast).map(v => AddSingleDependency(ast.depTree, v.selection.compositeSelections, v.rName, isDynamic)); 137 | 138 | return ast.depTree; 139 | } 140 | 141 | export function AddSingleDependencyByDerivedRelation(ast: DielAst, view: DerivedRelation) { 142 | const isDynamic = (rName: string) => { 143 | return IsRelationEvent(ast, rName); 144 | }; 145 | AddSingleDependency(ast.depTree, view.selection.compositeSelections, view.rName, isDynamic); 146 | } 147 | 148 | // CompositeSelection |CompositeSelectionFinal 149 | // the relation is dynamic if it's an event view 150 | export function AddSingleDependency ( 151 | depTree: DependencyTree, 152 | selection: CompositeSelection, 153 | rName: string, 154 | isDynamic: (rName: string) => boolean 155 | ) { 156 | // first add dependency one way, then the other way 157 | const dependsOn = addDependencyOneWay(depTree, selection, rName, isDynamic); 158 | addDependencyOtherWay(depTree, dependsOn, rName, isDynamic); 159 | } 160 | 161 | // CompositeSelection |CompositeSelectionFinal 162 | // incremental dep tree building 163 | function addDependencyOneWay(depTree: DependencyTree, selection: CompositeSelection, rName: string, 164 | isDynamic: (rName: string) => boolean 165 | ) { 166 | let dependsOn: string[] = []; 167 | selection.map(c => { 168 | const deps = getSelectionUnitDep(c.relation); 169 | dependsOn = deps.concat(dependsOn); 170 | }); 171 | depTree.set(rName, { 172 | relationName: rName, 173 | isDynamic: isDynamic(rName), 174 | dependsOn, 175 | isDependedBy: [] 176 | }); 177 | return dependsOn; 178 | } 179 | 180 | // recursive! 181 | // SelectionUnit | SelectionUnitFinal 182 | export function getSelectionUnitDep(s: SelectionUnit): string[] { 183 | if (!s.baseRelation) { 184 | return []; 185 | } 186 | const depsRaw = getRelationReferenceDep(s.baseRelation); 187 | let deps = depsRaw ? depsRaw : []; 188 | // the predicates on joins might have dependencies too... #FIXMELATER 189 | if (s.joinClauses) s.joinClauses.map(j => { 190 | const joinDepsRaw = getRelationReferenceDep(j.relation); 191 | if (joinDepsRaw) deps = deps.concat(joinDepsRaw); 192 | }); 193 | if (s.whereClause) { 194 | getExprDep(deps, s.whereClause); 195 | } 196 | return deps; 197 | } 198 | 199 | export function getRelationReferenceDep(ref: RelationReference): string[] | null { 200 | switch (ref.relationReferenceType) { 201 | case RelationReferenceType.Direct: { 202 | const r = ref as RelationReferenceDirect; 203 | return [r.relationName]; 204 | } 205 | case RelationReferenceType.Subquery: { 206 | const r = ref as RelationReferenceSubquery; 207 | let acc: string[] = []; 208 | r.subquery.compositeSelections.map(c => { 209 | acc = acc.concat(getSelectionUnitDep(c.relation)); 210 | }); 211 | return acc; 212 | } 213 | } 214 | } 215 | 216 | function getExprDep(depAcc: string[], e: ExprAst): void { 217 | if (!e) { 218 | debugger; 219 | } 220 | switch (e.exprType) { 221 | case ExprType.Relation: 222 | const relationExpr = e as ExprRelationAst; 223 | relationExpr.selection.compositeSelections.map(newE => { 224 | depAcc.push(...getSelectionUnitDep(newE.relation)); 225 | }); 226 | break; 227 | case ExprType.Func: 228 | const whereFuncExpr = (e as ExprFunAst); 229 | whereFuncExpr.args.map(newE => { 230 | getExprDep(depAcc, newE); 231 | }); 232 | break; 233 | case ExprType.Parenthesis: 234 | getExprDep(depAcc, (e as ExprParen).content); 235 | break; 236 | default: 237 | return; 238 | // do nothing for now 239 | } 240 | } 241 | 242 | function addDependencyOtherWay(depTree: DependencyTree, 243 | dependsOn: RelationNameType[], 244 | viewName: RelationNameType, 245 | isDynamic: (rName: string) => boolean 246 | ) { 247 | dependsOn.map(dO => { 248 | if (dO) { 249 | // avoid the case when its null 250 | // it's possible that these don't exist, if they are the leaves 251 | const dep = depTree.get(dO); 252 | if (!dep) { 253 | depTree.set(dO, { 254 | relationName: dO, 255 | dependsOn: [], 256 | isDependedBy: [viewName], 257 | isDynamic: isDynamic(dO), 258 | }); 259 | } else { 260 | dep.isDependedBy.push(viewName); 261 | } 262 | } 263 | }); 264 | } 265 | 266 | --------------------------------------------------------------------------------