├── .gitignore
├── LICENSE
├── README.md
├── client
├── README.md
├── package-lock.json
├── package.json
├── postbuild.js
├── public
│ ├── add-a-rewriting-rule.png
│ ├── browse-query-logs.png
│ ├── create-an-application.png
│ ├── download-config-file.png
│ ├── download-jdbc-driver.png
│ ├── favicon.ico
│ ├── formulate-a-rule-using-examples.png
│ ├── framework.png
│ ├── index.html
│ ├── logo192.png
│ ├── logo512.png
│ ├── manifest.json
│ ├── modify-config-file.png
│ ├── rewriting-rules.png
│ ├── robots.txt
│ ├── sign-in-with-google.png
│ └── varlsql-variables-definitions.png
└── src
│ ├── App.css
│ ├── App.js
│ ├── App.test.js
│ ├── constants
│ └── databaseOptions.js
│ ├── dashboard
│ ├── ApplicationSelect.js
│ ├── ApplicationTag.js
│ ├── Applications.js
│ ├── Dashboard.js
│ ├── FullLayout.js
│ ├── JDBCDrivers.js
│ ├── ManageApplicationModal.js
│ ├── QueryLogs.js
│ ├── QueryRewritingPath.js
│ ├── QuerySuggestionRewritingPath.js
│ ├── ReadMe.js
│ ├── RewritingRuleModal.js
│ ├── RewritingRules.js
│ ├── RuleFormulator.js
│ ├── RuleGraph.js
│ ├── RulesGraph.js
│ ├── Title.js
│ ├── ViewAssignedRules.js
│ └── listItems.js
│ ├── index.css
│ ├── index.js
│ ├── logo.svg
│ ├── mock-api
│ ├── generateRuleGraph.js
│ ├── generateRulesGraph.js
│ ├── listApplications.js
│ ├── listQueries.js
│ ├── listRules.js
│ ├── recommendRule.js
│ ├── recommendRules.js
│ ├── rewritingPath.js
│ └── suggestionRewritingPath.js
│ ├── reportWebVitals.js
│ ├── setupTests.js
│ └── userContext.js
├── core
├── __init__.py
├── profiler.py
├── query_patcher.py
├── query_rewriter.py
├── rule_generator.py
└── rule_parser.py
├── data
├── __init__.py
└── rules.py
├── examples
└── wetune.csv
├── experiments
├── Test_wetune.csv
├── Train_LeftOuterJoin_To_InnerJoin.csv
├── Train_Remove_1Useless_InnerJoin.csv
├── Train_Remove_1Useless_InnerJoin_Agg.csv
├── __init__.py
├── calcite_tests.csv
├── evaluate_generalize_rules.py
├── evaluate_suggest_rules.py
├── profile_suggest_rules.py
├── tpch_pg.md
├── tweets_cast_2q.csv
├── tweets_cast_3q.csv
├── tweets_cast_4q.csv
└── tweets_cast_5q.csv
├── management
├── app_manager.py
├── data_manager.py
├── query_manager.py
├── rule_manager.py
└── user_manager.py
├── misc
├── __init__.py
├── explore_mo_sql_parsing.py
└── explore_rule_parser.py
├── pub
└── framework.png
├── requirements.txt
├── schema
└── schema.sql
├── server
├── __init__.py
├── server.py
└── wsgi.py
├── sql
├── example1.sql
├── rule0_left.sql
├── rule0_left.svg
├── rule0_right.sql
├── rule0_right.svg
├── rule1_left.sql
├── rule1_left.svg
├── rule1_right.sql
├── rule1_right.svg
├── rule2_left.sql
├── rule2_left.svg
├── rule2_right.sql
├── rule2_right.svg
├── rule3_left.sql
├── rule3_left.svg
├── rule3_right.sql
├── rule3_right.svg
├── rule4_left.sql
├── rule4_left.svg
├── rule4_right.sql
├── rule4_right.svg
├── rule5_left.sql
├── rule5_left.svg
├── rule5_right.sql
├── rule5_right.svg
├── rule6_left.sql
├── rule6_left.svg
├── rule6_right.sql
├── rule6_right.svg
├── twitter_mysql_q1.sql
├── twitter_mysql_q1.svg
├── twitter_pg_q1.sql
├── twitter_pg_q1.svg
├── twitter_pg_q1_frag.sql
├── twitter_pg_q1_frag.svg
├── twitter_pg_q1_frag_p.sql
├── twitter_pg_q1_frag_p.svg
├── twitter_pg_q1_pp.sql
└── twitter_pg_q1_pp.svg
└── tests
├── __init__.py
├── integrate_rule_generator.py
├── string_util.py
├── test_query_patcher.py
├── test_query_rewriter.py
├── test_rule_generator.py
└── test_rule_parser.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | pip-wheel-metadata/
24 | share/python-wheels/
25 | *.egg-info/
26 | .installed.cfg
27 | *.egg
28 | MANIFEST
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | *.py,cover
51 | .hypothesis/
52 | .pytest_cache/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | target/
76 |
77 | # Jupyter Notebook
78 | .ipynb_checkpoints
79 |
80 | # IPython
81 | profile_default/
82 | ipython_config.py
83 |
84 | # pyenv
85 | .python-version
86 |
87 | # pipenv
88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
91 | # install all needed dependencies.
92 | #Pipfile.lock
93 |
94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow
95 | __pypackages__/
96 |
97 | # Celery stuff
98 | celerybeat-schedule
99 | celerybeat.pid
100 |
101 | # SageMath parsed files
102 | *.sage.py
103 |
104 | # Environments
105 | .env
106 | .venv
107 | env/
108 | venv/
109 | ENV/
110 | env.bak/
111 | venv.bak/
112 |
113 | # Spyder project settings
114 | .spyderproject
115 | .spyproject
116 |
117 | # Rope project settings
118 | .ropeproject
119 |
120 | # mkdocs documentation
121 | /site
122 |
123 | # mypy
124 | .mypy_cache/
125 | .dmypy.json
126 | dmypy.json
127 |
128 | # Pyre type checker
129 | .pyre/
130 |
131 | # ReactJS
132 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
133 |
134 | # dependencies
135 | client/node_modules
136 | client/.pnp
137 | .pnp.js
138 |
139 | # testing
140 | client/coverage
141 |
142 | # production
143 | client/build
144 | server/static
145 |
146 | # misc
147 | client/.gitignore
148 | .DS_Store
149 | .env.local
150 | .env.development.local
151 | .env.test.local
152 | .env.production.local
153 |
154 | npm-debug.log*
155 | yarn-debug.log*
156 | yarn-error.log*
157 |
158 |
159 | # database
160 | *.db
161 |
162 | # IDE
163 | .vscode/*
164 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # QueryBooster
2 |
3 | [https://querybooster.ics.uci.edu/](https://querybooster.ics.uci.edu/)
4 |
5 | ## News
6 | We gave a talk about this project at Cornell University, here is the [recording](https://drive.google.com/file/d/1JZt94sB2dTzICERljDoxNfcctCD_VBtx/view?usp=sharing).
7 |
8 | ## Introduction
9 | QueryBooster is a middleware-based **query rewriting** framework.
10 |
11 |
12 |
14 |
15 | QueryBooster intercepts SQL queries by customizing JDBC drivers used by applications (e.g., Tableau) and uses rules to rewrite the queries to semantically equivalent yet more efficient queries. The rewriting rules are designed by data experts who analyze slow queries and apply their domain knowledge and optimization expertise. QueryBooster can accelerate queries formulated by Tableau up to 100 times faster.
16 |
17 | ## JDBC Drivers
18 |
19 | The QueryBooster customized JDBC drivers repository are listed below:
20 |
21 | - [PostgreSQL JDBC Driver](https://github.com/ISG-ICS/smart-pgjdbc)
22 | - [MySQL JDBC Driver](https://github.com/ISG-ICS/smart-mysql-connector-j)
23 |
24 | ## Run QueryBooster
25 |
26 | #### Requirements
27 | - Python 3.9 +
28 | - NPM
29 |
30 | #### Set up Python Virtual Environement
31 | In `QueryBooster` folder,
32 | ```bash
33 | python3 -m venv venv
34 | source venv/bin/activate # Windows, .\venv\Scripts\activate
35 | pip install -r requirements.txt
36 | ```
37 |
38 | #### Compile and Deploy QueryBooster client
39 | In `QueryBooster` folder,
40 | ```bash
41 | cd client/
42 | npm install
43 | npm run build
44 | ```
45 |
46 | #### Run QueryBooster server
47 | In `QueryBooster` folder,
48 | ```bash
49 | cd server
50 |
51 | # Dev mode
52 | python wsgi.py
53 |
54 | # Server mode (only support Linux, Mac OS X)
55 | gunicorn 'wsgi:app'
56 | ```
57 |
58 | #### Access QueryBooster web interface
59 | Go to the link http://localhost:8000 to access the web interface.
60 |
61 |
62 | #### Test
63 | In `QueryBooster` folder,
64 | ```bash
65 | python3 -m pytest
66 | ```
67 |
68 | ## Publications
69 | 1. [**QueryBooster: Improving SQL Performance Using Middleware Services for Human-Centered Query Rewriting**](https://www.vldb.org/pvldb/vol16/p2911-bai.pdf) (published in VLDB 2023)
70 | 2. [**Demo of QueryBooster: Supporting Middleware-Based SQL Query Rewriting as a Service**](https://www.vldb.org/pvldb/vol16/p4038-bai.pdf) (published in VLDB 2023 Demo)
71 | The workloads we experimented on for the paper are the following:
72 | - Selected query pairs from WeTune: [Test_wetune.csv](https://github.com/ISG-ICS/QueryBooster/blob/main/experiments/Test_wetune.csv).
73 | - Query pairs from Calcite test suite: [calcite_tests.csv](https://github.com/ISG-ICS/QueryBooster/blob/main/experiments/calcite_tests.csv).
74 | - Tableau generated TPC-H queries (and their human-rewritten queries): [tpch_pg.md](https://github.com/ISG-ICS/QueryBooster/blob/main/experiments/tpch_pg.md).
75 |
--------------------------------------------------------------------------------
/client/README.md:
--------------------------------------------------------------------------------
1 | # Getting Started with Create React App
2 |
3 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
4 |
5 | ## Available Scripts
6 |
7 | In the project directory, you can run:
8 |
9 | ### `npm start`
10 |
11 | Runs the app in the development mode.\
12 | Open [http://localhost:3000](http://localhost:3000) to view it in your browser.
13 |
14 | The page will reload when you make changes.\
15 | You may also see any lint errors in the console.
16 |
17 | ### `npm test`
18 |
19 | Launches the test runner in the interactive watch mode.\
20 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.
21 |
22 | ### `npm run build`
23 |
24 | Builds the app for production to the `build` folder.\
25 | It correctly bundles React in production mode and optimizes the build for the best performance.
26 |
27 | The build is minified and the filenames include the hashes.\
28 | Your app is ready to be deployed!
29 |
30 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information.
31 |
32 | ### `npm run eject`
33 |
34 | **Note: this is a one-way operation. Once you `eject`, you can't go back!**
35 |
36 | If you aren't satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project.
37 |
38 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you're on your own.
39 |
40 | You don't have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn't feel obligated to use this feature. However we understand that this tool wouldn't be useful if you couldn't customize it when you are ready for it.
41 |
42 | ## Learn More
43 |
44 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started).
45 |
46 | To learn React, check out the [React documentation](https://reactjs.org/).
47 |
48 | ### Code Splitting
49 |
50 | This section has moved here: [https://facebook.github.io/create-react-app/docs/code-splitting](https://facebook.github.io/create-react-app/docs/code-splitting)
51 |
52 | ### Analyzing the Bundle Size
53 |
54 | This section has moved here: [https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size](https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size)
55 |
56 | ### Making a Progressive Web App
57 |
58 | This section has moved here: [https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app](https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app)
59 |
60 | ### Advanced Configuration
61 |
62 | This section has moved here: [https://facebook.github.io/create-react-app/docs/advanced-configuration](https://facebook.github.io/create-react-app/docs/advanced-configuration)
63 |
64 | ### Deployment
65 |
66 | This section has moved here: [https://facebook.github.io/create-react-app/docs/deployment](https://facebook.github.io/create-react-app/docs/deployment)
67 |
68 | ### `npm run build` fails to minify
69 |
70 | This section has moved here: [https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify](https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify)
71 |
--------------------------------------------------------------------------------
/client/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "client",
3 | "version": "0.1.0",
4 | "private": true,
5 | "dependencies": {
6 | "@ebay/nice-modal-react": "^1.2.7",
7 | "@emotion/react": "^11.8.2",
8 | "@emotion/styled": "^11.8.1",
9 | "@mui/icons-material": "^5.5.0",
10 | "@mui/material": "^5.5.0",
11 | "@mui/styled-engine-sc": "^5.4.2",
12 | "@react-oauth/google": "^0.11.0",
13 | "@testing-library/jest-dom": "^5.16.2",
14 | "@testing-library/react": "^12.1.4",
15 | "@testing-library/user-event": "^13.5.0",
16 | "ace-builds": "^1.24.1",
17 | "axios": "^0.26.1",
18 | "dagre": "^0.8.5",
19 | "diff": "^5.1.0",
20 | "fontsource-roboto": "^4.0.0",
21 | "github-markdown-css": "^5.5.1",
22 | "react": "^18.3.1",
23 | "react-ace": "^10.1.0",
24 | "react-dom": "^18.3.1",
25 | "react-markdown": "^9.0.1",
26 | "react-router-dom": "^6.2.2",
27 | "react-scripts": "5.0.0",
28 | "react-syntax-highlighter": "^15.5.0",
29 | "reactflow": "^11.1.2",
30 | "recharts": "^2.1.9",
31 | "rehype-raw": "^7.0.0",
32 | "sql-formatter": "^13.0.0",
33 | "styled-components": "^5.3.3",
34 | "web-vitals": "^2.1.4"
35 | },
36 | "scripts": {
37 | "start": "react-scripts start",
38 | "build": "react-scripts build",
39 | "postbuild": "node postbuild.js",
40 | "test": "react-scripts test",
41 | "eject": "react-scripts eject"
42 | },
43 | "eslintConfig": {
44 | "extends": [
45 | "react-app",
46 | "react-app/jest"
47 | ]
48 | },
49 | "browserslist": {
50 | "production": [
51 | ">0.2%",
52 | "not dead",
53 | "not op_mini all"
54 | ],
55 | "development": [
56 | "last 1 chrome version",
57 | "last 1 firefox version",
58 | "last 1 safari version"
59 | ]
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/client/postbuild.js:
--------------------------------------------------------------------------------
1 | var path = require('path');
2 | const fs = require("fs");
3 |
4 | const targetSource = './build'; // Relative path to copy files from
5 | const targetDestination = '../server/static'; // Relative path to copy files to
6 |
7 | /**
8 | * Remove directory recursively
9 | * @param {string} dir_path
10 | * @see https://stackoverflow.com/a/42505874
11 | */
12 | function rimraf(dir_path) {
13 | if (fs.existsSync(dir_path)) {
14 | fs.readdirSync(dir_path).forEach(function(entry) {
15 | var entry_path = path.join(dir_path, entry);
16 | if (fs.lstatSync(entry_path).isDirectory()) {
17 | rimraf(entry_path);
18 | } else {
19 | fs.unlinkSync(entry_path);
20 | }
21 | });
22 | fs.rmdirSync(dir_path);
23 | }
24 | }
25 |
26 | /**
27 | * Copy a file
28 | * @param {string} source
29 | * @param {string} target
30 | * @see https://stackoverflow.com/a/26038979
31 | */
32 | function copyFileSync(source, target) {
33 | var targetFile = target;
34 | // If target is a directory a new file with the same name will be created
35 | if (fs.existsSync(target)) {
36 | if (fs.lstatSync(target).isDirectory()) {
37 | targetFile = path.join(target, path.basename(source));
38 | }
39 | }
40 | fs.writeFileSync(targetFile, fs.readFileSync(source));
41 | }
42 |
43 | /**
44 | * Copy a folder recursively
45 | * @param {string} source
46 | * @param {string} target
47 | * @see https://stackoverflow.com/a/26038979
48 | */
49 | function copyFolderRecursiveSync(source, target, root = false) {
50 | var files = [];
51 | // Check if folder needs to be created or integrated
52 | var targetFolder = root ? target : path.join(target, path.basename(source));
53 | if (!fs.existsSync(targetFolder)) {
54 | fs.mkdirSync(targetFolder);
55 | }
56 | // Copy
57 | if (fs.lstatSync(source).isDirectory()) {
58 | files = fs.readdirSync(source);
59 | files.forEach(function (file) {
60 | var curSource = path.join(source, file);
61 | if (fs.lstatSync(curSource).isDirectory()) {
62 | copyFolderRecursiveSync(curSource, targetFolder);
63 | } else {
64 | copyFileSync(curSource, targetFolder);
65 | }
66 | });
67 | }
68 | }
69 |
70 | // Calculate absolute paths using the relative paths we defined at the top
71 | const sourceFolder = path.resolve(targetSource);
72 | const destinationFolder = path.resolve(targetDestination);
73 |
74 | // Remove destination folder if it exists to clear it
75 | if (fs.existsSync(destinationFolder)) {
76 | rimraf(destinationFolder)
77 | }
78 |
79 | // Copy the build over
80 | copyFolderRecursiveSync(sourceFolder, destinationFolder, true)
--------------------------------------------------------------------------------
/client/public/add-a-rewriting-rule.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/add-a-rewriting-rule.png
--------------------------------------------------------------------------------
/client/public/browse-query-logs.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/browse-query-logs.png
--------------------------------------------------------------------------------
/client/public/create-an-application.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/create-an-application.png
--------------------------------------------------------------------------------
/client/public/download-config-file.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/download-config-file.png
--------------------------------------------------------------------------------
/client/public/download-jdbc-driver.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/download-jdbc-driver.png
--------------------------------------------------------------------------------
/client/public/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/favicon.ico
--------------------------------------------------------------------------------
/client/public/formulate-a-rule-using-examples.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/formulate-a-rule-using-examples.png
--------------------------------------------------------------------------------
/client/public/framework.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/framework.png
--------------------------------------------------------------------------------
/client/public/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | React App
28 |
29 |
30 | You need to enable JavaScript to run this app.
31 |
32 |
42 |
43 |
44 |
--------------------------------------------------------------------------------
/client/public/logo192.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/logo192.png
--------------------------------------------------------------------------------
/client/public/logo512.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/logo512.png
--------------------------------------------------------------------------------
/client/public/manifest.json:
--------------------------------------------------------------------------------
1 | {
2 | "short_name": "React App",
3 | "name": "Create React App Sample",
4 | "icons": [
5 | {
6 | "src": "favicon.ico",
7 | "sizes": "64x64 32x32 24x24 16x16",
8 | "type": "image/x-icon"
9 | },
10 | {
11 | "src": "logo192.png",
12 | "type": "image/png",
13 | "sizes": "192x192"
14 | },
15 | {
16 | "src": "logo512.png",
17 | "type": "image/png",
18 | "sizes": "512x512"
19 | }
20 | ],
21 | "start_url": ".",
22 | "display": "standalone",
23 | "theme_color": "#000000",
24 | "background_color": "#ffffff"
25 | }
26 |
--------------------------------------------------------------------------------
/client/public/modify-config-file.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/modify-config-file.png
--------------------------------------------------------------------------------
/client/public/rewriting-rules.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/rewriting-rules.png
--------------------------------------------------------------------------------
/client/public/robots.txt:
--------------------------------------------------------------------------------
1 | # https://www.robotstxt.org/robotstxt.html
2 | User-agent: *
3 | Disallow:
4 |
--------------------------------------------------------------------------------
/client/public/sign-in-with-google.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/sign-in-with-google.png
--------------------------------------------------------------------------------
/client/public/varlsql-variables-definitions.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/client/public/varlsql-variables-definitions.png
--------------------------------------------------------------------------------
/client/src/App.css:
--------------------------------------------------------------------------------
1 | .App {
2 | text-align: center;
3 | }
4 |
5 | .App-logo {
6 | height: 40vmin;
7 | pointer-events: none;
8 | }
9 |
10 | @media (prefers-reduced-motion: no-preference) {
11 | .App-logo {
12 | animation: App-logo-spin infinite 20s linear;
13 | }
14 | }
15 |
16 | .App-header {
17 | background-color: #282c34;
18 | min-height: 100vh;
19 | display: flex;
20 | flex-direction: column;
21 | align-items: center;
22 | justify-content: center;
23 | font-size: calc(10px + 2vmin);
24 | color: white;
25 | }
26 |
27 | .App-link {
28 | color: #61dafb;
29 | }
30 |
31 | .add-position-marker {
32 | position: absolute;
33 | background-color: #30db58;
34 | }
35 |
36 | .add-all-marker {
37 | position: absolute;
38 | background-color: #98feb0;
39 | }
40 |
41 | .remove-marker {
42 | position: absolute;
43 | background-color: #ffa49b;
44 | }
45 |
46 | .replace-marker {
47 | position: absolute;
48 | background-color: #fff78e;
49 | }
50 |
51 | @keyframes App-logo-spin {
52 | from {
53 | transform: rotate(0deg);
54 | }
55 | to {
56 | transform: rotate(360deg);
57 | }
58 | }
59 |
60 | /* MarkdownPage.css */
61 | .markdown-container {
62 | padding: 20px;
63 | max-width: 800px;
64 | margin: auto;
65 | background-color: #fff; /* Light background */
66 | border: 1px solid #ddd;
67 | border-radius: 4px;
68 | text-align: left; /* Ensure left alignment for the container */
69 | }
70 |
71 | .markdown-body {
72 | box-sizing: border-box;
73 | min-width: 200px;
74 | max-width: 980px;
75 | margin: 0 auto;
76 | padding: 45px;
77 | color: #24292e; /* Default GitHub text color */
78 | background-color: #fff; /* Light background */
79 | text-align: left; /* Ensure left alignment for the content */
80 | }
81 |
82 | @media (max-width: 767px) {
83 | .markdown-body {
84 | padding: 15px;
85 | }
86 | }
87 |
88 |
--------------------------------------------------------------------------------
/client/src/App.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import logo from './logo.svg';
3 | import './App.css';
4 | import Dashboard from './dashboard/Dashboard';
5 |
6 | function App() {
7 | document.title = 'QueryBooster';
8 |
9 | return (
10 |
27 | );
28 | }
29 |
30 | export default App;
31 |
--------------------------------------------------------------------------------
/client/src/App.test.js:
--------------------------------------------------------------------------------
1 | import { render, screen } from '@testing-library/react';
2 | import App from './App';
3 |
4 | test('renders learn react link', () => {
5 | render( modal.hide()}
62 | TransitionProps={{
63 | onExited: () => modal.remove(),
64 | }}
65 | maxWidth={'sm'}
66 | >
67 | Enable Rule for Application
68 |
69 |
70 |
71 | Select...
72 | {applications.map((app) => (
73 | {app.name}
74 | ))}
75 |
76 |
77 |
78 |
79 | Confirm
80 |
81 |
82 | );
83 | });
84 |
85 | export default ApplicationSelect;
--------------------------------------------------------------------------------
/client/src/dashboard/ApplicationTag.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import axios from 'axios';
3 | import { useModal } from '@ebay/nice-modal-react';
4 | import ApplicationSelect from './ApplicationSelect';
5 | import {userContext} from '../userContext';
6 |
7 | function AppTagCell({ruleId: initialRuleId, tags: initialApps }) {
8 | const [ruleId, setRule] = React.useState(initialRuleId);
9 | const [apps, setApps] = React.useState(initialApps);
10 | // Set up a state for providing forceUpdate function
11 | const [, updateState] = React.useState();
12 | const forceUpdate = React.useCallback(() => updateState({}), []);
13 |
14 | const applicationSelectModal = useModal(ApplicationSelect);
15 |
16 | const user = React.useContext(userContext);
17 |
18 | function handleSelect(selectedApplication) {
19 | if (selectedApplication) {
20 | // post enableRule request to server
21 | axios.post('/enableRule', {'rule': {'id': ruleId}, 'app': selectedApplication})
22 | .then(function (response) {
23 | console.log('[/enableRule] -> response:');
24 | console.log(response);
25 | setApps([...apps, {'app_id': selectedApplication.id, 'app_name': selectedApplication.name}]);
26 | forceUpdate();
27 | })
28 | .catch(function (error) {
29 | console.log('[/enableRule] -> error:');
30 | console.log(error);
31 | // TODO - alter the entered application name doest not exist
32 | });
33 | }
34 | }
35 |
36 | const handleAddApp = React.useCallback(() => {
37 | applicationSelectModal.show({user}).then((selectedApplication) => {
38 | console.log("[ApplicationTag] selectedApplication = ");
39 | console.log(selectedApplication);
40 | handleSelect(selectedApplication);
41 | });
42 | }, [applicationSelectModal]);
43 |
44 | function handleRemoveApp(app) {
45 | // post disableRule request to server
46 | axios.post('/disableRule', {'rule': {'id': ruleId}, 'app': {'id': app.app_id, 'name': app.app_name}})
47 | .then(function (response) {
48 | console.log('[/disableRule] -> response:');
49 | console.log(response);
50 | const updatedApps = apps.filter((a) => a !== app);
51 | setApps(updatedApps);
52 | forceUpdate();
53 | })
54 | .catch(function (error) {
55 | console.log('[/disableRule] -> error:');
56 | console.log(error);
57 | });
58 | }
59 |
60 | return (
61 |
62 | {apps.map((app) => (
63 |
64 | {app.app_name}
65 | handleRemoveApp(app)}
68 | >
69 | x
70 |
71 |
72 | ))}
73 | +
74 |
75 | );
76 | }
77 |
78 | export default AppTagCell;
79 |
--------------------------------------------------------------------------------
/client/src/dashboard/Applications.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import axios from 'axios';
3 | import AddIcon from '@mui/icons-material/Add';
4 | import Button from '@mui/material/Button';
5 | import Fab from '@mui/material/Fab';
6 | import NiceModal from '@ebay/nice-modal-react';
7 | import Table from '@mui/material/Table';
8 | import TableBody from '@mui/material/TableBody';
9 | import TableCell from '@mui/material/TableCell';
10 | import TableContainer from '@mui/material/TableContainer';
11 | import TableHead from '@mui/material/TableHead';
12 | import TableRow from '@mui/material/TableRow';
13 | import Title from './Title';
14 | import { Box } from '@mui/material';
15 | import {userContext} from '../userContext';
16 | import ManageApplicationModal from './ManageApplicationModal';
17 | import ViewAssignedRules from './ViewAssignedRules';
18 | import FullLayout from './FullLayout';
19 |
20 | export default function Applications() {
21 | // Set up a state for list of apps
22 | const [applications, setApplications] = React.useState([]);
23 | // Set up a state for providing forceUpdate function
24 | const [, updateState] = React.useState();
25 | const forceUpdate = React.useCallback(() => updateState({}), []);
26 |
27 | const user = React.useContext(userContext);
28 |
29 | // initial loading applications for the current user from server
30 | const listApplications = () => {
31 | console.log('[/listApplications] -> request:');
32 | console.log(' user_id: ' + user.id);
33 | // post listApplications request to server
34 | axios.post('/listApplications', { 'user_id': user.id })
35 | .then(function (response) {
36 | console.log('[/listApplications] -> response:');
37 | console.log(response);
38 | // update the state for list of applications
39 | setApplications(response.data);
40 | forceUpdate();
41 | })
42 | .catch(function (error) {
43 | console.log('[/listApplications] -> error:');
44 | console.log(error);
45 | forceUpdate();
46 | });
47 | };
48 |
49 | // call listApplications() only once after initial rendering
50 | React.useEffect(() => { listApplications() }, [user]);
51 |
52 | // handle click on view assigned rules for this app
53 | const viewAssignedRules = (app) => {
54 | NiceModal.show(ViewAssignedRules, {user_id: user.id, app: app})
55 | .then((res) => {
56 | console.log(res);
57 | listApplications(user);
58 | });
59 | }
60 |
61 | // handle click on the delete of an application
62 | const handleDelete = (app) => {
63 | // post deleteApplication request to server
64 | console.log(app)
65 | axios.post('/deleteApplication', {user_id: user.id, app: app})
66 | .then(function (response) {
67 | console.log('[/deleteApplication] -> response:');
68 | console.log(response);
69 | listApplications(user);
70 | })
71 | .catch(function (error) {
72 | console.log('[/deleteApplication] -> error:');
73 | console.log(error);
74 | listApplications(user);
75 | });
76 | };
77 |
78 | // handle click on add AND edit an application
79 | const manageAppModal = (app) => {
80 | NiceModal.show(ManageApplicationModal, {user_id: user.id, app: app})
81 | .then((res) => {
82 | console.log(res);
83 | listApplications(user);
84 | });
85 | };
86 |
87 | return (
88 |
89 | Applications
90 |
91 |
92 |
93 |
94 | ID
95 | Name
96 | GUID
97 | Assigned Rules
98 | Edit
99 | Delete
100 |
101 |
102 |
103 | {applications.map((app) => (
104 |
105 | {app.id}
106 | {app.name}
107 | {app.guid}
108 |
109 | viewAssignedRules(app)} >View
110 |
111 |
112 | manageAppModal(app)}>Edit
113 |
114 |
115 | handleDelete(app)}>Delete
116 |
117 |
118 | ))}
119 |
120 |
121 |
122 |
123 | manageAppModal()}>
124 |
126 |
127 |
128 | );
129 | }
--------------------------------------------------------------------------------
/client/src/dashboard/FullLayout.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import { Box } from '@mui/material';
3 |
4 | export default function FullLayout({ children }) {
5 | return (
6 |
7 | {children}
8 |
9 | );
10 | }
11 |
--------------------------------------------------------------------------------
/client/src/dashboard/JDBCDrivers.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import Table from '@mui/material/Table';
3 | import TableBody from '@mui/material/TableBody';
4 | import TableCell from '@mui/material/TableCell';
5 | import TableHead from '@mui/material/TableHead';
6 | import TableRow from '@mui/material/TableRow';
7 | import Button from "@mui/material/Button";
8 | import Title from './Title';
9 | import FullLayout from './FullLayout';
10 |
11 | export default function JDBCDrivers() {
12 |
13 | const jdbcDrivers = [
14 | { database: "postgresql",
15 | version: "v42.3.3",
16 | download: "postgresql-42.3.3-SNAPSHOT.jar",
17 | href: "https://github.com/ISG-ICS/smart-pgjdbc/releases/download/smart_v42.3.3_new/postgresql-42.3.3-SNAPSHOT.jar",
18 | config: {
19 | download: "smart-pgjdbc.config",
20 | href: "https://github.com/ISG-ICS/smart-pgjdbc/releases/download/smart_v42.3.3_new/smart-pgjdbc.config",
21 | }
22 | },
23 | { database: "mysql",
24 | version: "v8.0.28",
25 | download: "mysql-connector-java-8.0.28-SNAPSHOT.jar",
26 | href: "https://github.com/ISG-ICS/smart-mysql-connector-j/releases/download/smart_v8.0.28/mysql-connector-java-8.0.28-SNAPSHOT.jar"
27 | }
28 | ];
29 |
30 | const onDownload = (download, href) => {
31 | const link = document.createElement("a");
32 | link.download = download;
33 | link.href = href;
34 | link.click();
35 | };
36 |
37 | return (
38 |
39 | JDBC Drivers
40 |
41 |
42 |
43 | Database
44 | Driver Version
45 | JDBC Driver
46 | Config File
47 |
48 |
49 |
50 | {jdbcDrivers.map((driver) => (
51 |
52 | {driver.database}
53 | {driver.version}
54 |
55 | onDownload(driver.download, driver.href)} variant="contained" color="primary">
56 | Download
57 |
58 |
59 |
60 | onDownload(driver.config.download, driver.config.href)} variant="contained" color="secondary">
61 | Config File
62 |
63 |
64 |
65 | ))}
66 |
67 |
68 |
69 | );
70 | }
71 |
--------------------------------------------------------------------------------
/client/src/dashboard/ManageApplicationModal.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import axios from 'axios';
3 | import Box from '@mui/material/Box';
4 | import Button from '@mui/material/Button';
5 | import Dialog from '@mui/material/Dialog';
6 | import DialogContent from '@mui/material/DialogContent';
7 | import DialogTitle from '@mui/material/DialogTitle';
8 | import Grid from '@mui/material/Grid';
9 | import NiceModal, { useModal } from '@ebay/nice-modal-react';
10 | import TextField from '@mui/material/TextField';
11 |
12 |
13 | const ManageApplicationModal = NiceModal.create(({user_id, app=null}) => {
14 | const modal = useModal();
15 | // Set up states for an application
16 | const isNewApplication = !app;
17 | const [name, setName] = React.useState(isNewApplication ? "" : app.name);
18 |
19 | const onNameChange = (event) => {
20 | setName(event.target.value);
21 | };
22 |
23 | const onAddOrEdit = () => {
24 | if (name !== null && name.replace(/\s/g, '').length ) {
25 |
26 | // check if user is authenticated before sending the request
27 | if (!user_id) {
28 | alert("Please log in to save the application.");
29 | return;
30 | }
31 |
32 | // post saveApplication request to server
33 | const request = {
34 | 'name': name,
35 | 'id': isNewApplication ? -1 : app.id,
36 | 'user_id': user_id
37 | };
38 | console.log('[/saveApplication] -> request:');
39 | console.log(request);
40 | axios.post('/saveApplication', request)
41 | .then(function (response) {
42 | console.log('[/saveApplication] -> response:');
43 | console.log(response);
44 | modal.resolve(response);
45 | modal.hide();
46 | })
47 | .catch(function (error) {
48 | console.log('[/saveApplication] -> error:');
49 | console.log(error);
50 | });
51 | }
52 | }
53 |
54 | return (
55 | modal.hide()}
58 | TransitionProps={{
59 | onExited: () => modal.remove(),
60 | }}
61 | fullWidth
62 | maxWidth={'lg'}
63 | >
64 | {isNewApplication ? "Add Application" : "Edit Application"}
65 |
66 |
67 |
68 |
69 |
70 |
72 |
73 |
75 |
76 | Save
77 |
78 |
79 |
80 |
81 |
82 |
83 | );
84 | });
85 |
86 | export default ManageApplicationModal;
--------------------------------------------------------------------------------
/client/src/dashboard/QueryLogs.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import axios from 'axios';
3 | import Link from '@mui/material/Link';
4 | import NiceModal from '@ebay/nice-modal-react';
5 | import Table from '@mui/material/Table';
6 | import TableBody from '@mui/material/TableBody';
7 | import TableCell from '@mui/material/TableCell';
8 | import TableContainer from '@mui/material/TableContainer';
9 | import TableHead from '@mui/material/TableHead';
10 | import TableRow from '@mui/material/TableRow';
11 | import Title from './Title';
12 | import defaultQueriesData from '../mock-api/listQueries';
13 | import QueryRewritingPath from './QueryRewritingPath';
14 | import QuerySuggestionRewritingPath from './QuerySuggestionRewritingPath';
15 | import SyntaxHighlighter from 'react-syntax-highlighter';
16 | import { vs } from 'react-syntax-highlighter/dist/esm/styles/hljs';
17 | import {userContext} from '../userContext';
18 | import FullLayout from './FullLayout';
19 |
20 |
21 | export default function QueryLogs() {
22 | // Set up a state for list of queries
23 | const [queries, setQueries] = React.useState([]);
24 | // Set up a state for providing forceUpdate function
25 | const [, updateState] = React.useState();
26 | const forceUpdate = React.useCallback(() => updateState({}), []);
27 |
28 | const user = React.useContext(userContext);
29 |
30 | // initial loading queries from server
31 | const listQueries = (_page) => {
32 | // post listQueries request to server
33 | axios.post('/listQueries', {page: _page, 'user_id': user.id})
34 | .then(function (response) {
35 | console.log('[/listQueries] -> response:');
36 | console.log(response);
37 | // update the state for list of queries
38 | setQueries(response.data);
39 | })
40 | .catch(function (error) {
41 | console.log('[/listQueries] -> error:');
42 | console.log(error);
43 | // mock the result
44 | console.log(defaultQueriesData);
45 | setQueries(defaultQueriesData);
46 | });
47 | };
48 |
49 | // call listQueries() only once after initial rendering
50 | React.useEffect(() => {listQueries(0)}, [user]);
51 |
52 | // show a query's rewriting path
53 | const showQueryRewritingPath = (query) => {
54 | console.log(query);
55 | NiceModal.show(QueryRewritingPath, {queryId: query.id});
56 | };
57 |
58 | // show a query's suggestion rewriting path
59 | const showQuerySuggestionRewritingPath = (query) => {
60 | console.log(query);
61 | NiceModal.show(QuerySuggestionRewritingPath, {user: user, queryId: query.id});
62 | };
63 |
64 | return (
65 |
66 | Query Logs
67 |
68 |
69 |
70 |
71 | ID
72 | App
73 | Timestamp
74 | Rewritten
75 | Before Latency(s)
76 | After Latency(s)
77 | SQL
78 | Suggestion
79 | Suggested Latency(s)
80 |
81 |
82 |
83 | {queries.map((query) => (
84 |
85 | {query.id}
86 | {query.app_name}
87 | {query.timestamp}
88 |
89 | {query.rewritten == 'YES' ? (
90 | {query.rewritten}
97 | )}
98 |
99 | {query.before_latency/1000}
100 | {query.after_latency/1000}
101 |
102 |
103 | {query.sql}
104 |
105 |
106 |
107 | {query.suggestion == 'YES' ? (
108 | {query.suggestion}
115 | )}
116 |
117 | {query.suggested_latency/1000}
118 |
119 | ))}
120 |
121 |
122 |
123 | {/*
127 | );
128 | }
129 |
--------------------------------------------------------------------------------
/client/src/dashboard/QueryRewritingPath.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import axios from 'axios';
3 | import Box from '@mui/material/Box';
4 | import Button from '@mui/material/Button';
5 | import Dialog from '@mui/material/Dialog';
6 | import DialogActions from '@mui/material/DialogActions';
7 | import DialogContent from '@mui/material/DialogContent';
8 | import DialogContentText from '@mui/material/DialogContentText';
9 | import DialogTitle from '@mui/material/DialogTitle';
10 | import Paper from '@mui/material/Paper';
11 | import Stack from '@mui/material/Stack';
12 | import { styled } from '@mui/material/styles';
13 | import NiceModal, { useModal } from '@ebay/nice-modal-react';
14 | import Title from './Title';
15 | import defaultRewritingPathData from '../mock-api/rewritingPath';
16 | import SyntaxHighlighter from 'react-syntax-highlighter';
17 | import { vs } from 'react-syntax-highlighter/dist/esm/styles/hljs';
18 |
19 | const Item = styled(Paper)(({ theme }) => ({
20 | backgroundColor: theme.palette.mode === 'dark' ? '#1A2027' : '#fff',
21 | ...theme.typography.body2,
22 | padding: theme.spacing(1),
23 | textAlign: 'left',
24 | color: theme.palette.text.primary,
25 | }));
26 |
27 | const QueryRewritingPath = NiceModal.create(({ queryId }) => {
28 | const modal = useModal();
29 | // Set up a state for rewritingPath
30 | const [rewritingPath, setRewritingPath] = React.useState({rewritings:[]});
31 | // Set up a state for providing forceUpdate function
32 | const [, updateState] = React.useState();
33 | const forceUpdate = React.useCallback(() => updateState({}), []);
34 |
35 | // initial loading rewritings from server
36 | const getRewritingPath = (_queryId) => {
37 | // post rewritingPath request to server
38 | axios.post('/rewritingPath', {queryId: _queryId})
39 | .then(function (response) {
40 | console.log('[/rewritingPath] -> response:');
41 | console.log(response);
42 | // update the state for rewritingPath
43 | setRewritingPath(response.data);
44 | })
45 | .catch(function (error) {
46 | console.log('[/rewritingPath] -> error:');
47 | console.log(error);
48 | // mock the result
49 | console.log(defaultRewritingPathData);
50 | setRewritingPath(defaultRewritingPathData);
51 | });
52 | };
53 |
54 | // call rewritePath() only once after initial rendering
55 | React.useEffect(() => {getRewritingPath(queryId)}, []);
56 |
57 | return (
58 | modal.hide()}
61 | TransitionProps={{
62 | onExited: () => modal.remove(),
63 | }}
64 | >
65 | Query Rewriting Path
66 |
67 |
68 |
69 |
70 | -
71 |
72 | {rewritingPath.original_sql}
73 |
74 |
75 | {rewritingPath.rewritings.map((rewriting) => (
76 |
77 | - {rewriting.rule}
78 | -
79 |
80 | {rewriting.rewritten_sql}
81 |
82 |
83 |
84 | ))}
85 |
86 |
87 |
88 |
89 |
90 | );
91 | });
92 |
93 | export default QueryRewritingPath;
94 |
--------------------------------------------------------------------------------
/client/src/dashboard/QuerySuggestionRewritingPath.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import axios from 'axios';
3 | import Box from '@mui/material/Box';
4 | import Button from '@mui/material/Button';
5 | import Dialog from '@mui/material/Dialog';
6 | import DialogActions from '@mui/material/DialogActions';
7 | import DialogContent from '@mui/material/DialogContent';
8 | import DialogContentText from '@mui/material/DialogContentText';
9 | import DialogTitle from '@mui/material/DialogTitle';
10 | import Paper from '@mui/material/Paper';
11 | import Stack from '@mui/material/Stack';
12 | import { styled } from '@mui/material/styles';
13 | import NiceModal, { useModal } from '@ebay/nice-modal-react';
14 | import Title from './Title';
15 | import defaultSuggestionRewritingPathData from '../mock-api/suggestionRewritingPath';
16 | import SyntaxHighlighter from 'react-syntax-highlighter';
17 | import { vs } from 'react-syntax-highlighter/dist/esm/styles/hljs';
18 | import {userContext} from "../userContext";
19 | import RewritingRuleModal from "./RewritingRuleModal";
20 |
21 |
22 | const Item = styled(Paper)(({ theme }) => ({
23 | backgroundColor: theme.palette.mode === 'dark' ? '#1A2027' : '#fff',
24 | ...theme.typography.body2,
25 | padding: theme.spacing(1),
26 | textAlign: 'left',
27 | color: theme.palette.text.primary,
28 | }));
29 |
30 | const QuerySuggestionRewritingPath = NiceModal.create(({ user, queryId }) => {
31 | const modal = useModal();
32 | // Set up a state for rewritingPath
33 | const [suggestionRewritingPath, setSuggestionRewritingPath] = React.useState({rewritings:[]});
34 | // Set up a state for providing forceUpdate function
35 | const [, updateState] = React.useState();
36 | const forceUpdate = React.useCallback(() => updateState({}), []);
37 |
38 | // initial loading suggestion rewritings from server
39 | const getSuggestionRewritingPath = (_queryId) => {
40 | // post suggestionRewritingPath request to server
41 | axios.post('/suggestionRewritingPath', {queryId: _queryId})
42 | .then(function (response) {
43 | console.log('[/suggestionRewritingPath] -> response:');
44 | console.log(response);
45 | // update the state for suggestionRewritingPath
46 | setSuggestionRewritingPath(response.data);
47 | })
48 | .catch(function (error) {
49 | console.log('[/suggestionRewritingPath] -> error:');
50 | console.log(error);
51 | // mock the result
52 | console.log(defaultSuggestionRewritingPathData);
53 | setSuggestionRewritingPath(defaultSuggestionRewritingPathData);
54 | });
55 | };
56 |
57 | // call getSuggestionRewritePath() only once after initial rendering
58 | React.useEffect(() => {getSuggestionRewritingPath(queryId)}, []);
59 |
60 | // handle click on Add to mine button
61 | const AddOrEditRewritingRule = (rule_id) => {
62 | console.log('[/fetchRule] -> request:');
63 | console.log(' rule_id: ' + rule_id);
64 | // post fetchRule request to server
65 | axios.post('/fetchRule', {'rule_id': rule_id})
66 | .then(function (response) {
67 | console.log('[/fetchRule] -> response:');
68 | console.log(response);
69 | const fetchedRule = response.data;
70 | // make sure the fetchedRule is a new rule to RewritingRuleModal
71 | // TODO - refine this logic later
72 | fetchedRule['id'] = -1;
73 | fetchedRule['name'] = fetchedRule['name'] + ' (Suggested)';
74 | // fetching the rule succeeds, open the RewritingRuleModal to add/edit the rule
75 | NiceModal.show(RewritingRuleModal, {user_id: user.id, rule: fetchedRule})
76 | .then((res) => {
77 | console.log(res);
78 | });
79 | })
80 | .catch(function (error) {
81 | console.log('[/fetchRule] -> error:');
82 | console.log(error);
83 | // fetching the rule fails, do nothing
84 | });
85 |
86 | };
87 |
88 | return (
89 | modal.hide()}
92 | TransitionProps={{
93 | onExited: () => modal.remove(),
94 | }}
95 | >
96 | Query Suggestion Rewriting Path
97 |
98 |
99 |
100 |
101 | -
102 |
103 | {suggestionRewritingPath.original_sql}
104 |
105 |
106 | {suggestionRewritingPath.rewritings.map((rewriting) => (
107 |
108 | -
109 |
110 | - {rewriting.rule}
111 | - {rewriting.rule_user_email}
112 | AddOrEditRewritingRule(rewriting.rule_id)} >Add to mine
113 |
114 |
115 | -
116 |
117 | {rewriting.rewritten_sql}
118 |
119 |
120 |
121 | ))}
122 |
123 |
124 |
125 |
126 |
127 | );
128 | });
129 |
130 | export default QuerySuggestionRewritingPath;
131 |
--------------------------------------------------------------------------------
/client/src/dashboard/ReadMe.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import ReactMarkdown from 'react-markdown';
3 | import rehypeRaw from 'rehype-raw';
4 | import 'github-markdown-css/github-markdown.css';
5 |
6 | // Static readme content (in Markdown)
7 | const readmeContent = `
8 | # QueryBooster
9 |
10 | [**https://github.com/ISG-ICS/QueryBooster**](https://github.com/ISG-ICS/QueryBooster)
11 |
12 | QueryBooster is a middleware-based **query rewriting** framework.
13 |
14 | 
15 |
16 | QueryBooster intercepts SQL queries by customizing JDBC drivers used by applications (e.g., Tableau) and uses rules to rewrite the queries to semantically equivalent yet more efficient queries. The rewriting rules are designed by data experts who analyze slow queries and apply their domain knowledge and optimization expertise. QueryBooster can accelerate queries formulated by Tableau up to 100 times faster.
17 |
18 | ## Start to use **QueryBooster**
19 |
20 | 1. Sign in through the top-right \`SIGN IN WITH GOOGLE\` button.
21 |
22 | 
23 |
24 | 2. Create an **Application** from the \`Applications\` page. (Take down the \`GUID\` value of your application.)
25 |
26 | 
27 |
28 | 3. Download the *customized* JDBC Driver using the \`DOWNLOAD\` button from the \`JDBC Drivers\` page, and put it in your application's required directory.
29 |
30 | 
31 |
32 | 4. Download the **config file** for your JDBC Driver using the \`CONFIG FILE\` button from the \`JDBC Drivers\` page, and put it in your home directory.
33 |
34 | 
35 |
36 | 5. Modify the **config file** by input \`your application's GUID\`.
37 |
38 | 
39 |
40 | Now, you can start using **QueryBooster** to
41 |
42 | ## Browse \`Query Logs\` for your application
43 |
44 | 
45 |
46 | ## Add \`Rewriting Rules\` to accelerate your queries
47 |
48 | 
49 |
50 | #### Either using the \`VarSQL\` Rule Language
51 |
52 | 
53 |
54 | #### Or provding examples to the \`Rule Formulator\` to automatically generate rules
55 |
56 | 
57 |
58 | ## \`VarSQL\` Rule Language
59 |
60 | The syntax of VarSQL to define a rewriting rule is as follows:
61 |
62 | \`\`\`
63 | [Rule] ::= [Pattern] / [Constraints] --> [Replacement] / [Actions].
64 | \`\`\`
65 |
66 | - The “Pattern” and “Replacement” components define how a query is matched and rewritten into a new query.
67 | - The “Constraints” component defines additional conditions that cannot be specified by a pattern such as
68 | schema-dependent conditions.
69 | - The “Actions” component defines extra operations that the replacement cannot express, such as replacing a table’s references with another table’s.
70 |
71 |
72 | The main idea of using VarSQL to define a rule’s pattern is to extend the SQL language with variables.
73 | A variable in a SQL query pattern can represent an existing SQL element such as a table, a column, a value, an expression, a predicate, a sub-query, etc.
74 | In this way, a user can formulate a query pattern as easily as writing a normal SQL query.
75 | The only difference is that, using VarSQL, one can use a variable to represent a specific SQL element so that the pattern can match a broad set of SQL queries.
76 | The pattern and replacement in a VarSQL rule have to be a full or partial SQL query optionally variablized.
77 | The variables and their matching conditions are defined in the following Table.
78 |
79 | 
80 |
81 | `;
82 |
83 | const ReadMe = () => {
84 | return (
85 |
86 |
87 | {readmeContent}
88 |
89 |
119 | Rewriting Rules
120 |
121 |
122 |
123 |
124 | ID
125 | Name
126 | Pattern
127 | Rewrite
128 | Enabled Apps
129 | Edit
130 | Delete
131 |
132 |
133 |
134 | {rules.map((rule) => (
135 |
136 | {rule.id}
137 | {rule.name}
138 |
139 |
140 | {rule.pattern}
141 |
142 |
143 |
144 |
145 | {rule.rewrite}
146 |
147 |
148 |
149 | {/* handleChange(event, rule)}
152 | inputProps={{ 'aria-label': 'controlled' }} /> */}
153 |
155 |
156 | AddOrEditRewritingRule(rule)} >Edit
157 |
158 |
159 | handleDelete(rule)} >Delete
160 |
161 |
162 | ))}
163 |
164 |
165 |
166 | {/*
170 | AddOrEditRewritingRule()}>
171 |
173 |
174 |
175 | );
176 | }
177 |
--------------------------------------------------------------------------------
/client/src/dashboard/RuleGraph.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import axios from 'axios';
3 | import dagre from 'dagre';
4 | import Dialog from '@mui/material/Dialog';
5 | import DialogContent from '@mui/material/DialogContent';
6 | import DialogTitle from '@mui/material/DialogTitle';
7 | import Divider from '@mui/material/Divider';
8 | import NiceModal, { useModal } from '@ebay/nice-modal-react';
9 | import Paper from '@mui/material/Paper';
10 | import Popover from '@mui/material/Popover';
11 | import Stack from '@mui/material/Stack';
12 | import ReactFlow, { Controls, MiniMap, useNodesState, useEdgesState, MarkerType } from 'reactflow';
13 | import 'reactflow/dist/style.css';
14 | import SyntaxHighlighter from 'react-syntax-highlighter';
15 | import { vs } from 'react-syntax-highlighter/dist/esm/styles/hljs';
16 |
17 | import defaultRuleGraphData from '../mock-api/generateRuleGraph';
18 |
19 | const nodeWidth = 300;
20 | const nodeHeight = 500;
21 |
22 | const RuleGraph = NiceModal.create(({ rewriteExample }) => {
23 | const modal = useModal();
24 | // Set up states for ReactFlow
25 | const [nodes, setNodes, onNodesChange] = useNodesState([]);
26 | const [edges, setEdges, onEdgesChange] = useEdgesState([]);
27 | // Set up states for PopOver
28 | const [anchorEl, setAnchorEl] = React.useState(null);
29 | const [popRule, setPopRule] = React.useState({});
30 | // Set up a state for providing forceUpdate function
31 | const [, updateState] = React.useState();
32 | const forceUpdate = React.useCallback(() => updateState({}), []);
33 |
34 | // initial loading rule graph from server
35 | const getRuleGraph = (_rewriteExample) => {
36 | if (_rewriteExample['q0'] != "" && _rewriteExample['q1'] != "") {
37 | // post generateRuleGraph request to server
38 | axios.post('/generateRuleGraph', _rewriteExample)
39 | .then(function (response) {
40 | console.log('[/generateRuleGraph] -> response:');
41 | console.log(response);
42 | // update the states for ruleGraph
43 | computeNodesAndEdges(response.data);
44 | })
45 | .catch(function (error) {
46 | console.log('[/generateRuleGraph] -> error:');
47 | console.log(error);
48 | // mock the result
49 | console.log(defaultRuleGraphData);
50 | computeNodesAndEdges(defaultRuleGraphData);
51 | });
52 | }
53 | };
54 |
55 | // compute the nodes and edges for the ruleGraph
56 | const computeNodesAndEdges = (_ruleGraphData) => {
57 | // loop rules in _ruleGraph to build nodes list
58 | let _rules = _ruleGraphData['rules'];
59 | let _nodes = [];
60 | for (var i = 0; i < _rules.length; i ++) {
61 | let _rule = _rules[i];
62 | let _node = {
63 | 'id': _rule['id'],
64 | 'data': {
65 | label:
66 | (
67 | <>
68 |
69 |
70 | {_rule['pattern']}
71 |
72 |
73 |
74 | ),
75 | rule: _rule
76 | },
77 | 'position': { x: 0, y: 0 }
78 | };
79 | // append new node to nodes
80 | _nodes.push(_node);
81 | }
82 | // loop relations in _ruleGraph to build edges list
83 | let _relations = _ruleGraphData['relations'];
84 | let _edges = [];
85 | for (var i = 0; i < _relations.length; i ++) {
86 | let _relation = _relations[i];
87 | let _edge = {
88 | 'id': i,
89 | 'source': _relation['parentRuleId'],
90 | 'target': _relation['childRuleId'],
91 | 'markerEnd': {
92 | type: MarkerType.Arrow,
93 | },
94 | };
95 | // append new edge to edges
96 | _edges.push(_edge);
97 | }
98 | // layout the nodes
99 | let { _nodes: layoutedNodes, _edges: layoutedEdges } = layoutNodesAndEdges(_nodes, _edges, 'LR');
100 | // update states for ReactFlow
101 | setNodes(_nodes);
102 | setEdges(_edges);
103 | };
104 |
105 | const dagreGraph = new dagre.graphlib.Graph();
106 | dagreGraph.setDefaultEdgeLabel(() => ({}));
107 |
108 | const layoutNodesAndEdges = (_nodes, _edges, _direction = 'TB') => {
109 | const isHorizontal = _direction === 'LR';
110 | dagreGraph.setGraph({ rankdir: _direction });
111 |
112 | _nodes.forEach((_node) => {
113 | dagreGraph.setNode(_node.id, { width: nodeWidth, height: nodeHeight });
114 | });
115 |
116 | _edges.forEach((_edge) => {
117 | dagreGraph.setEdge(_edge.source, _edge.target);
118 | });
119 |
120 | dagre.layout(dagreGraph);
121 |
122 | _nodes.forEach((_node) => {
123 | const nodeWithPosition = dagreGraph.node(_node.id);
124 | _node.targetPosition = isHorizontal ? 'left' : 'top';
125 | _node.sourcePosition = isHorizontal ? 'right' : 'bottom';
126 |
127 | // We are shifting the dagre node position (anchor=center center) to the top left
128 | // so it matches the React Flow node anchor point (top left).
129 | _node.position = {
130 | x: nodeWithPosition.x - nodeWidth / 2,
131 | y: nodeWithPosition.y - nodeHeight / 2,
132 | };
133 |
134 | return _node;
135 | });
136 |
137 | return { nodes: _nodes, edges: _edges };
138 | };
139 |
140 | const onNodeClick = (_event, _node) => {
141 | setAnchorEl(_event.currentTarget);
142 | setPopRule(_node.data.rule);
143 | };
144 |
145 | const handleClose = () => {
146 | setAnchorEl(null);
147 | };
148 | const open = Boolean(anchorEl);
149 |
150 | React.useEffect(() => {getRuleGraph(rewriteExample);}, []);
151 |
152 | return (
153 | modal.hide()}
156 | TransitionProps={{
157 | onExited: () => modal.remove(),
158 | }}
159 | fullWidth
160 | maxWidth={'xl'}
161 | >
162 | Rule Graph
163 |
164 |
165 |
180 |
185 |
186 | {popRule['pattern']}
187 |
188 |
189 |
190 |
191 | {popRule['rewrite']}
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 | );
200 | });
201 |
202 | export default RuleGraph;
203 |
--------------------------------------------------------------------------------
/client/src/dashboard/RulesGraph.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import axios from 'axios';
3 | import dagre from 'dagre';
4 | import Dialog from '@mui/material/Dialog';
5 | import DialogContent from '@mui/material/DialogContent';
6 | import DialogTitle from '@mui/material/DialogTitle';
7 | import Divider from '@mui/material/Divider';
8 | import NiceModal, { useModal } from '@ebay/nice-modal-react';
9 | import Paper from '@mui/material/Paper';
10 | import Popover from '@mui/material/Popover';
11 | import Stack from '@mui/material/Stack';
12 | import ReactFlow, { Controls, MiniMap, useNodesState, useEdgesState, MarkerType } from 'reactflow';
13 | import 'reactflow/dist/style.css';
14 | import SyntaxHighlighter from 'react-syntax-highlighter';
15 | import { vs } from 'react-syntax-highlighter/dist/esm/styles/hljs';
16 |
17 | import defaultRulesGraphData from '../mock-api/generateRulesGraph';
18 |
19 | const nodeWidth = 300;
20 | const nodeHeight = 500;
21 |
22 | const RulesGraph = NiceModal.create(({ rewriteExamples, database }) => {
23 | const modal = useModal();
24 | // Set up states for ReactFlow
25 | const [nodes, setNodes, onNodesChange] = useNodesState([]);
26 | const [edges, setEdges, onEdgesChange] = useEdgesState([]);
27 | // Set up states for PopOver
28 | const [anchorEl, setAnchorEl] = React.useState(null);
29 | const [popRule, setPopRule] = React.useState({});
30 | // Set up a state for providing forceUpdate function
31 | const [, updateState] = React.useState();
32 | const forceUpdate = React.useCallback(() => updateState({}), []);
33 |
34 | // initial loading rules graph from server
35 | const getRulesGraph = (_rewriteExamples, _database) => {
36 | if (_rewriteExamples.length >= 1 && _rewriteExamples[0].q0 != "" && _rewriteExamples[0].q1 != "") {
37 | // post generateRulesGraph request to server
38 | axios.post('/generateRulesGraph', {'examples': _rewriteExamples, 'database': _database})
39 | .then(function (response) {
40 | console.log('[/generateRulesGraph] -> response:');
41 | console.log(response);
42 | // update the states for rulesGraph
43 | computeNodesAndEdges(response.data);
44 | })
45 | .catch(function (error) {
46 | console.log('[/generateRulesGraph] -> error:');
47 | console.log(error);
48 | // mock the result
49 | console.log(defaultRulesGraphData);
50 | computeNodesAndEdges(defaultRulesGraphData);
51 | });
52 | }
53 | };
54 |
55 | // compute the nodes and edges for the rulesGraph
56 | const computeNodesAndEdges = (_rulesGraphData) => {
57 | // loop rules in _ruleGraph to build nodes list
58 | let _rules = _rulesGraphData['rules'];
59 | let _nodes = [];
60 | for (var i = 0; i < _rules.length; i ++) {
61 | let _rule = _rules[i];
62 | let _node = {
63 | 'id': _rule['id'],
64 | 'data': {
65 | label:
66 | (
67 | <>
68 |
69 |
70 | {_rule['pattern']}
71 |
72 |
73 |
74 | ),
75 | rule: _rule
76 | },
77 | 'position': { x: 0, y: 0 }
78 | };
79 | // append new node to nodes
80 | _nodes.push(_node);
81 | }
82 | // loop relations in _rulesGraph to build edges list
83 | let _relations = _rulesGraphData['relations'];
84 | let _edges = [];
85 | for (var i = 0; i < _relations.length; i ++) {
86 | let _relation = _relations[i];
87 | let _edge = {
88 | 'id': i,
89 | 'source': _relation['parentRuleId'],
90 | 'target': _relation['childRuleId'],
91 | 'markerEnd': {
92 | type: MarkerType.Arrow,
93 | },
94 | };
95 | // append new edge to edges
96 | _edges.push(_edge);
97 | }
98 | // layout the nodes
99 | let { _nodes: layoutedNodes, _edges: layoutedEdges } = layoutNodesAndEdges(_nodes, _edges, 'LR');
100 | // update states for ReactFlow
101 | setNodes(_nodes);
102 | setEdges(_edges);
103 | };
104 |
105 | const dagreGraph = new dagre.graphlib.Graph();
106 | dagreGraph.setDefaultEdgeLabel(() => ({}));
107 |
108 | const layoutNodesAndEdges = (_nodes, _edges, _direction = 'TB') => {
109 | const isHorizontal = _direction === 'LR';
110 | dagreGraph.setGraph({ rankdir: _direction });
111 |
112 | _nodes.forEach((_node) => {
113 | dagreGraph.setNode(_node.id, { width: nodeWidth, height: nodeHeight });
114 | });
115 |
116 | _edges.forEach((_edge) => {
117 | dagreGraph.setEdge(_edge.source, _edge.target);
118 | });
119 |
120 | dagre.layout(dagreGraph);
121 |
122 | _nodes.forEach((_node) => {
123 | const nodeWithPosition = dagreGraph.node(_node.id);
124 | _node.targetPosition = isHorizontal ? 'left' : 'top';
125 | _node.sourcePosition = isHorizontal ? 'right' : 'bottom';
126 |
127 | // We are shifting the dagre node position (anchor=center center) to the top left
128 | // so it matches the React Flow node anchor point (top left).
129 | _node.position = {
130 | x: nodeWithPosition.x - nodeWidth / 2,
131 | y: nodeWithPosition.y - nodeHeight / 2,
132 | };
133 |
134 | return _node;
135 | });
136 |
137 | return { nodes: _nodes, edges: _edges };
138 | };
139 |
140 | const onNodeClick = (_event, _node) => {
141 | setAnchorEl(_event.currentTarget);
142 | setPopRule(_node.data.rule);
143 | };
144 |
145 | const handleClose = () => {
146 | setAnchorEl(null);
147 | };
148 | const open = Boolean(anchorEl);
149 |
150 | React.useEffect(() => {getRulesGraph(rewriteExamples, database);}, []);
151 |
152 | return (
153 | modal.hide()}
156 | TransitionProps={{
157 | onExited: () => modal.remove(),
158 | }}
159 | fullWidth
160 | maxWidth={'xl'}
161 | >
162 | Rules Graph
163 |
164 |
165 |
180 |
185 |
186 | {popRule['pattern']}
187 |
188 |
189 |
190 |
191 | {popRule['rewrite']}
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 | );
200 | });
201 |
202 | export default RulesGraph;
203 |
--------------------------------------------------------------------------------
/client/src/dashboard/Title.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import PropTypes from 'prop-types';
3 | import Typography from '@mui/material/Typography';
4 |
5 | function Title(props) {
6 | return (
7 |
8 | {props.children}
9 |
10 | );
11 | }
12 |
13 | Title.propTypes = {
14 | children: PropTypes.node,
15 | };
16 |
17 | export default Title;
--------------------------------------------------------------------------------
/client/src/dashboard/ViewAssignedRules.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import axios from 'axios';
3 | import Dialog from '@mui/material/Dialog';
4 | import DialogContent from '@mui/material/DialogContent';
5 | import DialogTitle from '@mui/material/DialogTitle';
6 | import NiceModal, { useModal } from '@ebay/nice-modal-react';
7 | import Table from '@mui/material/Table';
8 | import TableBody from '@mui/material/TableBody';
9 | import TableCell from '@mui/material/TableCell';
10 | import TableContainer from '@mui/material/TableContainer';
11 | import TableHead from '@mui/material/TableHead';
12 | import TableRow from '@mui/material/TableRow';
13 | import SyntaxHighlighter from 'react-syntax-highlighter';
14 | import { vs } from 'react-syntax-highlighter/dist/esm/styles/hljs';
15 | import {userContext} from '../userContext';
16 |
17 |
18 | const ViewAssignedRules = NiceModal.create(({user_id, app}) => {
19 | const modal = useModal();
20 | // Set up a state for list of rules
21 | const [rules, setRules] = React.useState([]);
22 |
23 | // Set up a state for providing forceUpdate function
24 | const [, updateState] = React.useState();
25 | const forceUpdate = React.useCallback(() => updateState({}), []);
26 |
27 | const user = React.useContext(userContext);
28 |
29 | // initial loading rules from server
30 | const listRules = () => {
31 | console.log('[/listRules] -> request:');
32 | console.log(' user_id: ' + user_id);
33 | // post listRules request to server
34 | axios.post('/listRules', {'user_id': user_id, 'app_id': app.id})
35 | .then(function (response) {
36 | console.log('[/listRules] -> response:');
37 | console.log(response);
38 | // update the state for list of rules
39 | setRules(response.data);
40 | forceUpdate();
41 | })
42 | .catch(function (error) {
43 | console.log('[/listRules] -> error:');
44 | console.log(error);
45 | });
46 | };
47 |
48 | // call listRules() only once after initial rendering
49 | React.useEffect(() => {listRules()}, [user_id, app.id]);
50 |
51 | return (
52 | modal.hide()}
55 | TransitionProps={{
56 | onExited: () => modal.remove(),
57 | }}
58 | fullWidth
59 | maxWidth={'lg'}
60 | >
61 | Assigned Rules for {app.name}
62 |
63 |
64 |
65 |
66 |
67 | ID
68 | Name
69 | Pattern
70 | Rewrite
71 |
72 |
73 |
74 | {rules.map((rule) => (
75 |
76 | {rule.id}
77 | {rule.name}
78 |
79 |
80 | {rule.pattern}
81 |
82 |
83 |
84 |
85 | {rule.rewrite}
86 |
87 |
88 |
89 | ))}
90 |
91 |
92 |
93 |
94 |
95 | );
96 | });
97 |
98 | export default ViewAssignedRules;
--------------------------------------------------------------------------------
/client/src/dashboard/listItems.js:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import {Link} from "react-router-dom";
3 | import ListItemButton from '@mui/material/ListItemButton';
4 | import ListItemIcon from '@mui/material/ListItemIcon';
5 | import ListItemText from '@mui/material/ListItemText';
6 | import ListSubheader from '@mui/material/ListSubheader';
7 | import BorderColorIcon from '@mui/icons-material/BorderColor';
8 | import HistoryIcon from '@mui/icons-material/History';
9 | import NotesIcon from '@mui/icons-material/Notes';
10 | import CloudDownloadIcon from '@mui/icons-material/CloudDownload';
11 | import AssignmentIcon from '@mui/icons-material/Assignment';
12 | import AppRegistrationIcon from '@mui/icons-material/AppRegistration';
13 | import InfoIcon from '@mui/icons-material/Info';
14 |
15 | export const mainListItems = (
16 |
17 | {/*
18 |
19 |
21 | */}
23 |
24 |
25 |
27 |
29 |
30 |
31 |
33 |
35 |
36 |
37 |
39 |
41 |
42 |
43 |
45 |
47 |
48 |
49 |
51 |
53 |
54 |
55 |
57 |
59 |
60 | );
61 |
62 | export const secondaryListItems = (
63 |
64 |
65 | Saved reports
66 |
67 |
68 |
69 |
71 |
73 |
74 |
75 |
77 |
79 |
80 |
81 |
83 |
85 |
86 | );
87 |
--------------------------------------------------------------------------------
/client/src/index.css:
--------------------------------------------------------------------------------
1 | body {
2 | margin: 0;
3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
5 | sans-serif;
6 | -webkit-font-smoothing: antialiased;
7 | -moz-osx-font-smoothing: grayscale;
8 | }
9 |
10 | code {
11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
12 | monospace;
13 | }
14 |
--------------------------------------------------------------------------------
/client/src/index.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import ReactDOM from 'react-dom';
3 | import { BrowserRouter } from "react-router-dom";
4 | import './index.css';
5 | import App from './App';
6 | import NiceModal from '@ebay/nice-modal-react';
7 | import reportWebVitals from './reportWebVitals';
8 | import 'fontsource-roboto'
9 | import { GoogleOAuthProvider } from '@react-oauth/google';
10 |
11 | ReactDOM.render(
12 |
13 |
14 |
15 |
16 |
18 |
19 |
20 | ,
21 | document.getElementById('root')
22 | );
23 |
24 | // If you want to start measuring performance in your app, pass a function
25 | // to log results (for example: reportWebVitals(console.log))
26 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
27 | reportWebVitals();
28 |
--------------------------------------------------------------------------------
/client/src/logo.svg:
--------------------------------------------------------------------------------
1 | AS DATE)',
14 | 'rewrite': '',
15 | 'constraints': '',
16 | 'actions': '',
17 | 'level': 2,
18 | 'recommended': true
19 | }
20 | ],
21 | 'relations': [
22 | {'parentRuleId': '1', 'childRuleId': '2'}
23 | ]
24 | };
25 |
26 | export default defaultRuleGraphData;
--------------------------------------------------------------------------------
/client/src/mock-api/generateRulesGraph.js:
--------------------------------------------------------------------------------
1 | const defaultRulesGraphData = {
2 | 'rules': [
3 | {
4 | 'id': '1',
5 | 'pattern': 'CAST(created_at AS DATE)',
6 | 'rewrite': 'created_at',
7 | 'constraints': '',
8 | 'actions': '',
9 | 'level': 1
10 | },
11 | {
12 | 'id': '2',
13 | 'pattern': 'CAST( AS DATE)',
14 | 'rewrite': '',
15 | 'constraints': '',
16 | 'actions': '',
17 | 'level': 2,
18 | 'recommended': true
19 | },
20 | {
21 | 'id': '3',
22 | 'pattern': 'CAST(timestamp AS DATE)',
23 | 'rewrite': 'timestamp',
24 | 'constraints': '',
25 | 'actions': '',
26 | 'level': 1
27 | }
28 | ],
29 | 'relations': [
30 | {'parentRuleId': '1', 'childRuleId': '2'},
31 | {'parentRuleId': '3', 'childRuleId': '2'}
32 | ]
33 | };
34 |
35 | export default defaultRulesGraphData;
--------------------------------------------------------------------------------
/client/src/mock-api/listApplications.js:
--------------------------------------------------------------------------------
1 | const defaultApplicationsData = [
2 | {
3 | "id": 1,
4 | "name": "TwitterPg"
5 | },
6 | {
7 | "id": 2,
8 | "name": "TpchPg"
9 | },
10 | {
11 | "id": 3,
12 | "name": "TwitterMySQL"
13 | }
14 | ];
15 |
16 | export default defaultApplicationsData;
--------------------------------------------------------------------------------
/client/src/mock-api/listQueries.js:
--------------------------------------------------------------------------------
1 | const defaultQueriesData = [
2 | {
3 | "id": 1,
4 | "timestamp": "2022-10-12 16:36:03",
5 | "rewritten": 'YES',
6 | "before_latency": 35000,
7 | "after_latency": 3200,
8 | "sql": `SELECT SUM(1) AS "cnt:tweets_5460F7F804494E7CB9FD188E329004C1:ok",
9 | CAST("tweets"."state_name" AS TEXT) AS "state_name"
10 | FROM "public"."tweets" "tweets"
11 | WHERE ((CAST(DATE_TRUNC('QUARTER', CAST("tweets"."created_at" AS DATE)) AS DATE) IN ((TIMESTAMP '2016-04-01 00:00:00.000'), (TIMESTAMP '2016-07-01 00:00:00.000'), (TIMESTAMP '2016-10-01 00:00:00.000'), (TIMESTAMP '2017-01-01 00:00:00.000')))
12 | AND (STRPOS(CAST(LOWER(CAST(CAST("tweets"."text" AS TEXT) AS TEXT)) AS TEXT), CAST('iphone' AS TEXT)) > 0))
13 | GROUP BY 2`,
14 | "suggestion": "NO",
15 | "suggested_latency": -1000,
16 | "app_name": "TwitterPg"
17 | },
18 | {
19 | "id": 0,
20 | "timestamp": "2022-10-12 16:31:42",
21 | "rewritten": 'NO',
22 | "before_latency": 32000,
23 | "after_latency": 32000,
24 | "sql": `SELECT "lineitem"."l_shipmode" AS "l_shipmode",
25 | SUM("lineitem"."l_quantity") AS "sum:l_quantity:ok:1"
26 | FROM "public"."lineitem" AS "lineitem"
27 | WHERE STRPOS(CAST(LOWER(CAST("lineitem"."l_comment" AS TEXT)) AS TEXT), CAST('late' AS TEXT)) > 0
28 | GROUP BY 1`,
29 | "suggestion": "YES",
30 | "suggested_latency": (Math.round(32000*3200/35000)),
31 | "app_name": "TpchPg"
32 | }
33 | ];
34 |
35 | export default defaultQueriesData
--------------------------------------------------------------------------------
/client/src/mock-api/listRules.js:
--------------------------------------------------------------------------------
1 | const defaultRulesData = [
2 | {
3 | "id": 0,
4 | "key": "remove_max_distinct",
5 | "name": "Remove Max Distinct",
6 | "pattern": "MAX(DISTINCT )",
7 | "constraints": "",
8 | "rewrite": "MAX()",
9 | "actions": "",
10 | "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}]
11 | },
12 | {
13 | "id": 10,
14 | "key": "remove_cast_date",
15 | "name": "Remove Cast Date",
16 | "pattern": "CAST( AS DATE)",
17 | "constraints": "TYPE(x)=DATE",
18 | "rewrite": "",
19 | "actions": "",
20 | "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}, {"app_id": 3, "app_name": "TwitterMySQL"}]
21 | },
22 | {
23 | "id": 11,
24 | "key": "remove_cast_text",
25 | "name": "Remove Cast Text",
26 | "pattern": "CAST( AS TEXT)",
27 | "constraints": "TYPE(x)=TEXT",
28 | "rewrite": "",
29 | "actions": "",
30 | "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}, {"app_id": 3, "app_name": "TwitterMySQL"}]
31 | },
32 | {
33 | "id": 21,
34 | "key": "replace_strpos_lower",
35 | "name": "Replace Strpos Lower",
36 | "pattern": "STRPOS(LOWER(),'')>0",
37 | "constraints": "IS(y)=CONSTANT and\nTYPE(y)=STRING",
38 | "rewrite": " ILIKE '%%'",
39 | "actions": "",
40 | "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}, {"app_id": 3, "app_name": "TwitterMySQL"}]
41 | },
42 | {
43 | "id": 30,
44 | "key": "remove_self_join",
45 | "name": "Remove Self Join",
46 | "pattern": "select <> \nfrom ,\n \nwhere .=.\nand <>\n",
47 | "constraints": "UNIQUE(tb1, a1)",
48 | "rewrite": "select <> \nfrom \nwhere 1=1 \nand <>\n",
49 | "actions": "SUBSTITUTE(s1, t2, t1) and\nSUBSTITUTE(p1, t2, t1)",
50 | "enabled_apps": [{"app_id": 1, "app_name": "TwitterPg"}, {"app_id": 3, "app_name": "TwitterMySQL"}]
51 | },
52 | {
53 | "id": 101,
54 | "key": "remove_adddate",
55 | "name": "Remove Adddate",
56 | "pattern": "ADDDATE(, INTERVAL 0 SECOND)",
57 | "constraints": "",
58 | "rewrite": "",
59 | "actions": "",
60 | "enabled_apps": [{"app_id": 3, "app_name": "TwitterMySQL"}]
61 | },
62 | {
63 | "id": 102,
64 | "key": "remove_timestamp",
65 | "name": "Remove Timestamp",
66 | "pattern": " = TIMESTAMP()",
67 | "constraints": "TYPE(x)=STRING",
68 | "rewrite": " = ",
69 | "actions": "",
70 | "enabled_apps": [{"app_id": 3, "app_name": "TwitterMySQL"}]
71 | }
72 | ];
73 |
74 | export default defaultRulesData
--------------------------------------------------------------------------------
/client/src/mock-api/recommendRule.js:
--------------------------------------------------------------------------------
1 | const defaultRecommendRuleData =
2 | {
3 | "pattern": 'CAST( AS DATE)',
4 | "rewrite": ''
5 | };
6 |
7 | export default defaultRecommendRuleData;
--------------------------------------------------------------------------------
/client/src/mock-api/recommendRules.js:
--------------------------------------------------------------------------------
1 | const defaultRecommendRulesData = [
2 | {
3 | "pattern": "CAST( AS DATE)",
4 | "rewrite": ""
5 | },
6 | {
7 | "pattern": "STRPOS(UPPER(),'')>0",
8 | "rewrite": " ILIKE '%%'"
9 | }
10 | ];
11 |
12 | export default defaultRecommendRulesData;
--------------------------------------------------------------------------------
/client/src/mock-api/rewritingPath.js:
--------------------------------------------------------------------------------
1 | const defaultRewritingPathData =
2 | {
3 | "original_sql": `SELECT SUM(1),
4 | CAST(state_name AS TEXT)
5 | FROM tweets
6 | WHERE CAST(DATE_TRUNC('QUARTER',
7 | CAST(created_at AS DATE))
8 | AS DATE) IN
9 | ((TIMESTAMP '2016-10-01 00:00:00.000'),
10 | (TIMESTAMP '2017-01-01 00:00:00.000'),
11 | (TIMESTAMP '2017-04-01 00:00:00.000'))
12 | AND (STRPOS(text, 'iphone') > 0)
13 | GROUP BY 2`,
14 | "rewritings":
15 | [
16 | {
17 | "seq": 1,
18 | "rule": "Remove Cast Date",
19 | "rewritten_sql": `SELECT SUM(1),
20 | CAST(state_name AS TEXT)
21 | FROM tweets
22 | WHERE DATE_TRUNC('QUARTER', created_at)
23 | IN
24 | ((TIMESTAMP '2016-10-01 00:00:00.000'),
25 | (TIMESTAMP '2017-01-01 00:00:00.000'),
26 | (TIMESTAMP '2017-04-01 00:00:00.000'))
27 | AND (STRPOS(text, 'iphone') > 0)
28 | GROUP BY 2`
29 | },
30 | {
31 | "seq": 2,
32 | "rule": "Replace Strpos Lower",
33 | "rewritten_sql": `SELECT SUM(1),
34 | CAST(state_name AS TEXT)
35 | FROM tweets
36 | WHERE DATE_TRUNC('QUARTER', created_at)
37 | IN
38 | ((TIMESTAMP '2016-10-01 00:00:00.000'),
39 | (TIMESTAMP '2017-01-01 00:00:00.000'),
40 | (TIMESTAMP '2017-04-01 00:00:00.000'))
41 | AND text ILIKE '%iphone%'
42 | GROUP BY 2`
43 | }
44 | ]
45 | };
46 |
47 | export default defaultRewritingPathData;
--------------------------------------------------------------------------------
/client/src/mock-api/suggestionRewritingPath.js:
--------------------------------------------------------------------------------
1 | const defaultSuggestionRewritingPathData =
2 | {
3 | "original_sql": `SELECT "lineitem"."l_shipmode" AS "l_shipmode",
4 | SUM("lineitem"."l_quantity") AS "sum:l_quantity:ok:1"
5 | FROM "public"."lineitem" AS "lineitem"
6 | WHERE STRPOS(CAST(LOWER(CAST("lineitem"."l_comment" AS TEXT)) AS TEXT), CAST('late' AS TEXT)) > 0
7 | GROUP BY 1`,
8 | "rewritings":
9 | [
10 | {
11 | "seq": 1,
12 | "rule": "Remove Cast Text",
13 | "rule_id": 11,
14 | "rule_user_id": 102153741508111367852,
15 | "rule_user_email": "alice.vldb@gmail.com",
16 | "rewritten_sql": `SELECT "lineitem"."l_shipmode" AS "l_shipmode",
17 | SUM("lineitem"."l_quantity") AS "sum:l_quantity:ok:1"
18 | FROM "public"."lineitem" AS "lineitem"
19 | WHERE STRPOS(LOWER(CAST("lineitem"."l_comment" AS TEXT)), CAST('late' AS TEXT)) > 0
20 | GROUP BY 1`
21 | },
22 | {
23 | "seq": 2,
24 | "rule": "Remove Cast Text",
25 | "rule_id": 11,
26 | "rule_user_id": 102153741508111367852,
27 | "rule_user_email": "alice.vldb@gmail.com",
28 | "rewritten_sql": `SELECT "lineitem"."l_shipmode" AS "l_shipmode",
29 | SUM("lineitem"."l_quantity") AS "sum:l_quantity:ok:1"
30 | FROM "public"."lineitem" AS "lineitem"
31 | WHERE STRPOS(LOWER("lineitem"."l_comment"), CAST('late' AS TEXT)) > 0
32 | GROUP BY 1`
33 | },
34 | {
35 | "seq": 3,
36 | "rule": "Remove Cast Text",
37 | "rule_id": 11,
38 | "rule_user_id": 102153741508111367852,
39 | "rule_user_email": "alice.vldb@gmail.com",
40 | "rewritten_sql": `SELECT "lineitem"."l_shipmode" AS "l_shipmode",
41 | SUM("lineitem"."l_quantity") AS "sum:l_quantity:ok:1"
42 | FROM "public"."lineitem" AS "lineitem"
43 | WHERE STRPOS(LOWER("lineitem"."l_comment"), 'late') > 0
44 | GROUP BY 1`
45 | },
46 | {
47 | "seq": 4,
48 | "rule": "Replace Strpos Lower",
49 | "rule_id": 21,
50 | "rule_user_id": 102153741508111367852,
51 | "rule_user_email": "alice.vldb@gmail.com",
52 | "rewritten_sql": `SELECT "lineitem"."l_shipmode" AS "l_shipmode",
53 | SUM("lineitem"."l_quantity") AS "sum:l_quantity:ok:1"
54 | FROM "public"."lineitem" AS "lineitem"
55 | WHERE "lineitem"."l_comment" ILIKE '%late%'
56 | GROUP BY 1`
57 | }
58 | ]
59 | };
60 |
61 | export default defaultSuggestionRewritingPathData;
--------------------------------------------------------------------------------
/client/src/reportWebVitals.js:
--------------------------------------------------------------------------------
1 | const reportWebVitals = onPerfEntry => {
2 | if (onPerfEntry && onPerfEntry instanceof Function) {
3 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => {
4 | getCLS(onPerfEntry);
5 | getFID(onPerfEntry);
6 | getFCP(onPerfEntry);
7 | getLCP(onPerfEntry);
8 | getTTFB(onPerfEntry);
9 | });
10 | }
11 | };
12 |
13 | export default reportWebVitals;
14 |
--------------------------------------------------------------------------------
/client/src/setupTests.js:
--------------------------------------------------------------------------------
1 | // jest-dom adds custom jest matchers for asserting on DOM nodes.
2 | // allows you to do things like:
3 | // expect(element).toHaveTextContent(/react/i)
4 | // learn more: https://github.com/testing-library/jest-dom
5 | import '@testing-library/jest-dom';
6 |
--------------------------------------------------------------------------------
/client/src/userContext.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 |
3 | const userContext = React.createContext({user: {}});
4 |
5 | export { userContext };
--------------------------------------------------------------------------------
/core/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/core/__init__.py
--------------------------------------------------------------------------------
/core/profiler.py:
--------------------------------------------------------------------------------
1 | import time
2 |
3 | profiles = {}
4 |
5 | class Profiler:
6 |
7 | @staticmethod
8 | def onFunctionStart(function: str) -> None:
9 | if function not in profiles.keys():
10 | profiles[function] = {'calls': 0, 'total_time': 0}
11 | profiles[function]['calls'] += 1
12 | profiles[function]['timer'] = {'start': time.time()}
13 | return
14 |
15 | @staticmethod
16 | def onFunctionEnd(function: str) -> None:
17 | if function not in profiles.keys():
18 | return
19 | start_time = profiles[function]['timer']['start']
20 | del profiles[function]['timer']
21 | end_time = time.time()
22 | timing = end_time - start_time
23 | profiles[function]['total_time'] += timing
24 | return
25 |
26 | @staticmethod
27 | def show() -> dict:
28 | return profiles
29 |
--------------------------------------------------------------------------------
/core/query_patcher.py:
--------------------------------------------------------------------------------
1 | import re
2 |
3 |
4 | class QueryPatcher:
5 |
6 | # Patch the SQL query based on specific database
7 | #
8 | @staticmethod
9 | def patch(sql: str, database: str='postgresql') -> str:
10 | if database == 'postgresql':
11 | sql = QueryPatcher.patch_ilike(sql)
12 | sql = QueryPatcher.patch_timestamp(sql)
13 | return sql
14 |
15 | # Patch the SQL query with a trasnformation for ILIKE
16 | # Note: This is needed only because the SQL Parser (mo_sql_parsing)
17 | # we use treats ILIKE as a normal function instead of a predicate,
18 | # e.g., ILIKE("tweets"."text",\'%microsoft%\').
19 | # We need to use regex to transform it back to
20 | # "tweets"."text" ILIKE \'%microsoft%\'.
21 | # Demo: https://regex101.com/r/gkUZb4/3
22 | #
23 | @staticmethod
24 | def patch_ilike(sql: str) -> str:
25 | return re.sub(r"ILIKE\((.[^\,]*)\s*,\s*(.[^\)]*)\)", r"\1 ILIKE \2", sql)
26 |
27 | # Patch the SQL query with a transformation for TIMESTAMP (postgresql)
28 | # Note: PostgreSQL syntax for TIMESTAMP constant is the following:
29 | # TIMESTAMP '2017-04-01 00:00:00.000'
30 | # while the SQL Parser (mo_sql_parsing) serializes it as:
31 | # TIMESTAMP('2017-04-01 00:00:00.000')
32 | # Demo: https://regex101.com/r/ywmIUn/4
33 | #
34 | @staticmethod
35 | def patch_timestamp(sql: str) -> str:
36 | return re.sub(r"TIMESTAMP\(\s*(.[^\)]*)\s*\)", r"(TIMESTAMP \1)", sql)
37 |
--------------------------------------------------------------------------------
/data/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/data/__init__.py
--------------------------------------------------------------------------------
/experiments/Train_LeftOuterJoin_To_InnerJoin.csv:
--------------------------------------------------------------------------------
1 | q0,q1
2 | "SELECT *
3 | FROM posts
4 | LEFT OUTER JOIN participations
5 | ON participations.target_id = posts.id
6 | AND participations.target_type = 'post'
7 | WHERE participations.author_id = 1047","SELECT *
8 | FROM posts
9 | INNER JOIN participations
10 | ON participations.target_id = posts.id
11 | AND participations.target_type = 'post'
12 | WHERE participations.author_id = 1047"
13 | "SELECT *
14 | FROM posts0
15 | LEFT OUTER JOIN participations0
16 | ON participations0.target_id = posts0.id
17 | AND participations0.target_type = 'post'
18 | WHERE participations0.author_id = 1047","SELECT *
19 | FROM posts0
20 | INNER JOIN participations0
21 | ON participations0.target_id = posts0.id
22 | AND participations0.target_type = 'post'
23 | WHERE participations0.author_id = 1047"
24 | "SELECT target_id0
25 | FROM posts
26 | LEFT OUTER JOIN participations
27 | ON participations.target_id0 = posts.id0
28 | AND participations.target_type = 'post'
29 | WHERE participations.author_id = 1047","SELECT target_id0
30 | FROM posts
31 | INNER JOIN participations
32 | ON participations.target_id0 = posts.id0
33 | AND participations.target_type = 'post'
34 | WHERE participations.author_id = 1047"
35 | "SELECT target_id0
36 | FROM posts
37 | LEFT OUTER JOIN participations
38 | ON participations.target_id0 = posts.id0
39 | AND participations.target_type0 = 'post'
40 | WHERE participations.author_id0 = 1047","SELECT target_id0
41 | FROM posts
42 | INNER JOIN participations
43 | ON participations.target_id0 = posts.id0
44 | AND participations.target_type0 = 'post'
45 | WHERE participations.author_id0 = 1047"
46 | "SELECT *
47 | FROM posts
48 | LEFT OUTER JOIN participations
49 | ON participations.target_id = posts.id
50 | AND participations.author_id = 10470
51 | WHERE participations.target_type = 'post0'","SELECT *
52 | FROM posts
53 | INNER JOIN participations
54 | ON participations.target_id = posts.id
55 | AND participations.author_id = 10470
56 | WHERE participations.target_type = 'post0'"
57 | "SELECT *
58 | FROM posts
59 | LEFT OUTER JOIN participations
60 | ON participations.target_id = posts.id
61 | AND participations.author_id = 10470
62 | WHERE posts.interacted_at < '2020-04-28 06:05:28'","SELECT *
63 | FROM posts
64 | INNER JOIN participations
65 | ON participations.target_id = posts.id
66 | AND participations.author_id = 10470
67 | WHERE posts.interacted_at < '2020-04-28 06:05:28'"
--------------------------------------------------------------------------------
/experiments/Train_Remove_1Useless_InnerJoin.csv:
--------------------------------------------------------------------------------
1 | q0,q1
2 | "SELECT groups.id
3 | FROM groups
4 | INNER JOIN invited_groups
5 | ON groups.id = invited_groups.group_id
6 | WHERE invited_groups.invite_id = 1318 ","SELECT invited_groups.group_id
7 | FROM invited_groups
8 | WHERE invited_groups.invite_id = 1318 "
9 | "SELECT groups0.id
10 | FROM groups0
11 | INNER JOIN invited_groups0
12 | ON groups0.id = invited_groups0.group_id
13 | WHERE invited_groups0.invite_id = 1318 ","SELECT invited_groups0.group_id
14 | FROM invited_groups0
15 | WHERE invited_groups0.invite_id = 1318 "
16 | "SELECT groups.id0
17 | FROM groups
18 | INNER JOIN invited_groups
19 | ON groups.id0 = invited_groups.group_id0
20 | WHERE invited_groups.invite_id = 1318 ","SELECT invited_groups.group_id0
21 | FROM invited_groups
22 | WHERE invited_groups.invite_id = 1318 "
23 | "SELECT groups.id
24 | FROM groups
25 | INNER JOIN invited_groups
26 | ON groups.id = invited_groups.group_id
27 | WHERE invited_groups.invite_id0 = 13180 ","SELECT invited_groups.group_id
28 | FROM invited_groups
29 | WHERE invited_groups.invite_id0 = 13180"
--------------------------------------------------------------------------------
/experiments/Train_Remove_1Useless_InnerJoin_Agg.csv:
--------------------------------------------------------------------------------
1 | q0,q1
2 | "SELECT Count(*)
3 | FROM tags
4 | INNER JOIN tag_followings
5 | ON tags.id = tag_followings.tag_id
6 | WHERE tag_followings.user_id = 1 ","SELECT Count(*)
7 | FROM tag_followings
8 | WHERE tag_followings.user_id = 1"
9 | "SELECT Count(*)
10 | FROM t
11 | INNER JOIN tf
12 | ON t.id = tf.tag_id
13 | WHERE tf.user_id = 1 ","SELECT Count(*)
14 | FROM tf
15 | WHERE tf.user_id = 1"
16 | "SELECT Count(*)
17 | FROM tags
18 | INNER JOIN tag_followings
19 | ON tags.name = tag_followings.tag_name
20 | WHERE tag_followings.group_id = 1 ","SELECT Count(*)
21 | FROM tag_followings
22 | WHERE tag_followings.group_id = 1"
23 | "SELECT Count(1)
24 | FROM tags
25 | INNER JOIN tag_followings
26 | ON tags.id = tag_followings.tag_id
27 | WHERE tag_followings.user_id = 100","SELECT Count(1)
28 | FROM tag_followings
29 | WHERE tag_followings.user_id = 100"
--------------------------------------------------------------------------------
/experiments/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/experiments/__init__.py
--------------------------------------------------------------------------------
/experiments/tweets_cast_2q.csv:
--------------------------------------------------------------------------------
1 | q0,q1
2 | SELECT * FROM tweets WHERE CAST( AS DATE) = TIMESTAMP('2016-10-01 00:00:00.000'),SELECT * FROM tweets WHERE = TIMESTAMP('2016-10-01 00:00:00.000')
3 | SELECT * FROM tweets WHERE CAST(created_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM tweets WHERE created_at = TIMESTAMP '2016-10-01 00:00:00.000'
4 | SELECT * FROM tweets WHERE CAST(deleted_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM tweets WHERE deleted_at = TIMESTAMP '2016-10-01 00:00:00.000'
--------------------------------------------------------------------------------
/experiments/tweets_cast_3q.csv:
--------------------------------------------------------------------------------
1 | q0,q1
2 | SELECT * FROM tweets WHERE CAST( AS DATE) = TIMESTAMP(''),SELECT * FROM tweets WHERE = TIMESTAMP('')
3 | SELECT * FROM tweets WHERE CAST(created_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM tweets WHERE created_at = TIMESTAMP '2016-10-01 00:00:00.000'
4 | SELECT * FROM tweets WHERE CAST(deleted_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM tweets WHERE deleted_at = TIMESTAMP '2016-10-01 00:00:00.000'
5 | SELECT * FROM tweets WHERE CAST(created_at AS DATE) = TIMESTAMP '2018-10-01 00:00:00.000',SELECT * FROM tweets WHERE created_at = TIMESTAMP '2018-10-01 00:00:00.000'
--------------------------------------------------------------------------------
/experiments/tweets_cast_4q.csv:
--------------------------------------------------------------------------------
1 | q0,q1
2 | SELECT * FROM WHERE CAST( AS DATE) = TIMESTAMP(''),SELECT * FROM WHERE = TIMESTAMP('')
3 | SELECT * FROM tweets WHERE CAST(created_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM tweets WHERE created_at = TIMESTAMP '2016-10-01 00:00:00.000'
4 | SELECT * FROM tweets WHERE CAST(deleted_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM tweets WHERE deleted_at = TIMESTAMP '2016-10-01 00:00:00.000'
5 | SELECT * FROM tweets WHERE CAST(created_at AS DATE) = TIMESTAMP '2018-10-01 00:00:00.000',SELECT * FROM tweets WHERE created_at = TIMESTAMP '2018-10-01 00:00:00.000'
6 | SELECT * FROM users WHERE CAST(deleted_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM users WHERE deleted_at = TIMESTAMP '2016-10-01 00:00:00.000'
--------------------------------------------------------------------------------
/experiments/tweets_cast_5q.csv:
--------------------------------------------------------------------------------
1 | q0,q1
2 | SELECT <> FROM WHERE CAST( AS DATE) = TIMESTAMP(''),SELECT <> FROM WHERE = TIMESTAMP('')
3 | SELECT * FROM tweets WHERE CAST(created_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM tweets WHERE created_at = TIMESTAMP '2016-10-01 00:00:00.000'
4 | SELECT * FROM tweets WHERE CAST(deleted_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM tweets WHERE deleted_at = TIMESTAMP '2016-10-01 00:00:00.000'
5 | SELECT * FROM tweets WHERE CAST(created_at AS DATE) = TIMESTAMP '2018-10-01 00:00:00.000',SELECT * FROM tweets WHERE created_at = TIMESTAMP '2018-10-01 00:00:00.000'
6 | SELECT * FROM users WHERE CAST(deleted_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT * FROM users WHERE deleted_at = TIMESTAMP '2016-10-01 00:00:00.000'
7 | SELECT id FROM users WHERE CAST(deleted_at AS DATE) = TIMESTAMP '2016-10-01 00:00:00.000',SELECT id FROM users WHERE deleted_at = TIMESTAMP '2016-10-01 00:00:00.000'
--------------------------------------------------------------------------------
/management/app_manager.py:
--------------------------------------------------------------------------------
1 | import uuid
2 | import sys
3 | # append the path of the parent directory
4 | sys.path.append("..")
5 | from management.data_manager import DataManager
6 |
7 |
8 | class AppManager:
9 |
10 | def __init__(self, dm: DataManager) -> None:
11 | self.dm = dm
12 |
13 | def __del__(self):
14 | del self.dm
15 |
16 | def list_applications(self, user_id: str) -> list:
17 | applications = self.dm.list_applications(user_id)
18 | res = []
19 | for app in applications:
20 | res.append({
21 | 'id': app[0],
22 | 'name': app[1],
23 | 'guid': app[2]
24 | })
25 | return res
26 |
27 | def save_application(self, app: dict) -> bool:
28 | app['guid'] = str(uuid.uuid1())
29 | return self.dm.save_application(app)
30 |
31 | def delete_application(self, app: dict) -> bool:
32 | return self.dm.delete_application(app)
33 |
--------------------------------------------------------------------------------
/management/query_manager.py:
--------------------------------------------------------------------------------
1 | import sys
2 | # append the path of the parent directory
3 | sys.path.append("..")
4 | from management.data_manager import DataManager
5 | import json
6 |
7 |
8 | class QueryManager:
9 |
10 | def __init__(self, dm: DataManager) -> None:
11 | self.dm = dm
12 |
13 | def __del__(self):
14 | del self.dm
15 |
16 | def log_query(self, appguid: str, guid: str, original_query: str, rewritten_query: str, rewriting_path: list) -> None:
17 | self.dm.log_query(appguid, guid, original_query, rewritten_query, rewriting_path)
18 |
19 | def report_query(self, appguid: str, guid: str, query_time_ms: int) -> None:
20 | self.dm.report_query(appguid, guid, query_time_ms)
21 |
22 | def list_queries(self, user_id: str) -> list:
23 | queries = self.dm.list_queries(user_id)
24 | res = []
25 | for query in queries:
26 | res.append({
27 | 'id': query[0],
28 | 'timestamp': query[1],
29 | 'rewritten': query[2],
30 | 'before_latency': query[3],
31 | 'after_latency': query[4],
32 | 'sql': query[5],
33 | 'suggestion': query[6],
34 | 'suggested_latency': query[7],
35 | 'app_name': query[8]
36 | })
37 | return res
38 |
39 | def rewriting_path(self, query_id: str) -> dict:
40 | original_sql = self.dm.get_original_sql(query_id)
41 | res = {
42 | "original_sql": original_sql,
43 | "rewritings":[]
44 | }
45 | rewritings = self.dm.list_rewritings(query_id)
46 | for rewriting in rewritings:
47 | res["rewritings"].append({
48 | "seq": rewriting[0],
49 | "rule": rewriting[1],
50 | "rewritten_sql": rewriting[2]
51 | })
52 | return res
53 |
54 | def suggestion_rewriting_path(self, query_id: str) -> dict:
55 | original_sql = self.dm.get_original_sql(query_id)
56 | res = {
57 | "original_sql": original_sql,
58 | "rewritings":[]
59 | }
60 | rewritings = self.dm.list_suggestion_rewritings(query_id)
61 | for rewriting in rewritings:
62 | res["rewritings"].append({
63 | "seq": rewriting[0],
64 | "rule": rewriting[1],
65 | "rule_id": rewriting[2],
66 | "rule_user_id": rewriting[3],
67 | "rule_user_email": rewriting[4],
68 | "rewritten_sql": rewriting[5]
69 | })
70 | return res
71 |
72 | def fetch_query(self, guid: str) -> dict:
73 | query = self.dm.fetch_query(guid)
74 | return {
75 | 'id': query[0],
76 | 'rewritten': query[1],
77 | 'sql': query[2]
78 | }
79 |
80 | def log_query_suggestion(self, query_id: str, rewritten_query: str, rewriting_path: list) -> None:
81 | self.dm.log_query_suggestion(query_id, rewritten_query, rewriting_path)
82 |
--------------------------------------------------------------------------------
/management/user_manager.py:
--------------------------------------------------------------------------------
1 | import sys
2 | # append the path of the parent directory
3 | sys.path.append("..")
4 | from management.data_manager import DataManager
5 |
6 |
7 | class UserManager:
8 |
9 | def __init__(self, dm: DataManager) -> None:
10 | self.dm = dm
11 |
12 | def __del__(self):
13 | del self.dm
14 |
15 | def create_user(self, user: dict) -> bool:
16 | return self.dm.create_user(user)
17 |
--------------------------------------------------------------------------------
/misc/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/misc/__init__.py
--------------------------------------------------------------------------------
/misc/explore_mo_sql_parsing.py:
--------------------------------------------------------------------------------
1 | from typing import Any
2 | import mo_sql_parsing as mosql
3 | import json
4 | from pathlib import Path
5 |
6 |
7 | def sql_to_ast(sqlStr: str) -> Any:
8 | return mosql.parse(sqlStr)
9 |
10 | def ast_to_str(astObj: Any) -> str:
11 | return json.dumps(astObj)
12 |
13 | def ast_to_sql(astObj: Any) -> str:
14 | return mosql.format(astObj)
15 |
16 | def print_sql_str(title: str, sqlStr: str) -> None:
17 | astJson = sql_to_ast(sqlStr)
18 | print()
19 | print("==================================================")
20 | print(" " + title)
21 | print("==================================================")
22 | print("Original SQL: ")
23 | print("--------------------------------------------------")
24 | print(sqlStr)
25 | print("--------------------------------------------------")
26 | print("Parsed JSON: ")
27 | print("--------------------------------------------------")
28 | print(ast_to_str(astJson))
29 | print("--------------------------------------------------")
30 | print("Formated SQL: ")
31 | print("--------------------------------------------------")
32 | print(ast_to_sql(astJson))
33 | print()
34 |
35 | def explore_sql_files() -> None:
36 | sql_path = Path(__file__).parent / "../sql"
37 | sql_files = [f for f in sql_path.iterdir() if f.is_file() and f.suffix == '.sql']
38 | for sql_file in sql_files:
39 | with sql_file.open() as f:
40 | sql = sql_file.read_text()
41 | print_sql_str(sql_file.name, sql)
42 |
43 | def explore_rules() -> None:
44 | # Rule #1 -> pattern
45 | sqlStr = '''
46 | SELECT * FROM t0 WHERE CAST(e AS DATE)
47 | '''
48 | print_sql_str('Rule #1 -> pattern', sqlStr)
49 |
50 | # Rule #1 -> rewrite
51 | sqlStr = '''
52 | SELECT * FROM t0 WHERE e
53 | '''
54 | print_sql_str('Rule #1 -> rewrite', sqlStr)
55 |
56 | # Rule #2 -> pattern
57 | sqlStr = '''
58 | SELECT * FROM t0 WHERE STRPOS(LOWER(e), s) > 0
59 | '''
60 | print_sql_str('Rule #2 -> pattern', sqlStr)
61 |
62 | # Rule #2 -> rewrite
63 | sqlStr = '''
64 | SELECT * FROM t0 WHERE e ILIKE '%s%'
65 | '''
66 | print_sql_str('Rule #2 -> rewrite', sqlStr)
67 |
68 | # Rule #3 -> pattern
69 | sqlStr = '''
70 | SELECT s1
71 | FROM tab1 t1, tab2 t2
72 | WHERE t1.a1 = t2.a2
73 | AND p1
74 | '''
75 | print_sql_str('Rule #3 -> pattern', sqlStr)
76 |
77 | # Rule #3 -> rewrite
78 | sqlStr = '''
79 | SELECT s1
80 | FROM tab1 t1
81 | WHERE 1=1 AND p1
82 | '''
83 | print_sql_str('Rule #3 -> rewrite', sqlStr)
84 |
85 | # Rule #3' -> pattern
86 | sqlStr = '''
87 | SELECT s
88 | FROM t1, t2
89 | WHERE t1.a1 = t2.a2
90 | AND p
91 | '''
92 | print_sql_str('Rule #3\' -> pattern', sqlStr)
93 |
94 | # Rule #3' -> rewrite
95 | sqlStr = '''
96 | SELECT s
97 | FROM t1
98 | WHERE p
99 | '''
100 | print_sql_str('Rule #3\' -> rewrite', sqlStr)
101 |
102 | def explore_ast():
103 | sqlStr = '''
104 | SELECT * FROM t0 WHERE CAST(e AS DATE)
105 | '''
106 | sqlAST = sql_to_ast(sqlStr)
107 | print(sqlAST)
108 |
109 | if __name__ == '__main__':
110 | explore_rules()
111 | explore_sql_files()
112 | explore_ast()
113 |
--------------------------------------------------------------------------------
/misc/explore_rule_parser.py:
--------------------------------------------------------------------------------
1 | import sys
2 | # append the path of the parent directory
3 | sys.path.append("..")
4 | from core.rule_parser import RuleParser
5 | from data.rules import rules
6 | import json
7 |
8 | if __name__ == '__main__':
9 | for rule in rules:
10 | rule['pattern_json'], rule['rewrite_json'], rule['mapping'] = RuleParser.parse(rule['pattern'], rule['rewrite'])
11 | rule['constraints_json'] = RuleParser.parse_constraints(rule['constraints'], rule['mapping'])
12 | rule['actions_json'] = RuleParser.parse_actions(rule['actions'], rule['mapping'])
13 | print(json.dumps(rule, indent=2))
14 |
--------------------------------------------------------------------------------
/pub/framework.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/pub/framework.png
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | sqlparse
2 | configparser
3 | mo-sql-parsing
4 | pytest
5 | Flask
6 | gunicorn
--------------------------------------------------------------------------------
/schema/schema.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE IF NOT EXISTS rules(
2 | id INTEGER PRIMARY KEY,
3 | key VARCHAR(255) UNIQUE,
4 | name VARCHAR(2048) NOT NULL,
5 | pattern TEXT,
6 | constraints TEXT,
7 | rewrite TEXT,
8 | actions TEXT,
9 | user_id TEXT,
10 | CONSTRAINT fk_rules
11 | FOREIGN KEY (user_id)
12 | REFERENCES users(id)
13 | );
14 |
15 | CREATE TABLE IF NOT EXISTS internal_rules(
16 | rule_id INTEGER UNIQUE,
17 | pattern_json TEXT,
18 | constraints_json TEXT,
19 | rewrite_json TEXT,
20 | actions_json TEXT,
21 | CONSTRAINT fk_rules
22 | FOREIGN KEY (rule_id)
23 | REFERENCES rules(id)
24 | ON DELETE CASCADE
25 | );
26 |
27 | CREATE TABLE IF NOT EXISTS users(
28 | id TEXT PRIMARY KEY,
29 | email TEXT
30 | );
31 |
32 | CREATE TABLE IF NOT EXISTS applications(
33 | id INTEGER PRIMARY KEY,
34 | name TEXT,
35 | guid TEXT,
36 | user_id TEXT,
37 | CONSTRAINT fk_users
38 | FOREIGN KEY (user_id)
39 | REFERENCES users(id)
40 | ON DELETE CASCADE
41 | );
42 |
43 | CREATE TABLE IF NOT EXISTS enabled(
44 | application_id INTEGER,
45 | rule_id INTEGER,
46 | PRIMARY KEY (application_id, rule_id),
47 | CONSTRAINT fk_applications
48 | FOREIGN KEY (application_id)
49 | REFERENCES applications(id)
50 | ON DELETE CASCADE,
51 | CONSTRAINT fk_rules
52 | FOREIGN KEY (rule_id)
53 | REFERENCES rules(id)
54 | ON DELETE CASCADE
55 | );
56 |
57 | CREATE TABLE IF NOT EXISTS queries(
58 | id INTEGER PRIMARY KEY,
59 | guid TEXT,
60 | appguid TEXT,
61 | timestamp TEXT,
62 | query_time_ms REAL,
63 | original_sql TEXT,
64 | sql TEXT
65 | );
66 |
67 | CREATE TABLE IF NOT EXISTS rewriting_paths(
68 | query_id INTEGER,
69 | seq INTEGER,
70 | rule_id INTEGER,
71 | rewritten_sql TEXT,
72 | PRIMARY KEY (query_id, seq),
73 | CONSTRAINT fk_queries
74 | FOREIGN KEY (query_id)
75 | REFERENCES queries(id)
76 | ON DELETE CASCADE,
77 | CONSTRAINT fk_rules
78 | FOREIGN KEY (rule_id)
79 | REFERENCES rules(id)
80 | ON DELETE CASCADE
81 | );
82 |
83 | CREATE TABLE IF NOT EXISTS suggestions(
84 | query_id INTEGER UNIQUE,
85 | query_time_ms REAL,
86 | rewritten_sql TEXT,
87 | CONSTRAINT fk_queries
88 | FOREIGN KEY (query_id)
89 | REFERENCES queries(id)
90 | ON DELETE CASCADE
91 | );
92 |
93 | CREATE TABLE IF NOT EXISTS suggestion_rewriting_paths(
94 | query_id INTEGER,
95 | seq INTEGER,
96 | rule_id INTEGER,
97 | rewritten_sql TEXT,
98 | PRIMARY KEY (query_id, seq),
99 | CONSTRAINT fk_queries
100 | FOREIGN KEY (query_id)
101 | REFERENCES suggestions(query_id)
102 | ON DELETE CASCADE,
103 | CONSTRAINT fk_rules
104 | FOREIGN KEY (rule_id)
105 | REFERENCES rules(id)
106 | ON DELETE CASCADE
107 | );
108 |
109 | CREATE TABLE IF NOT EXISTS tables(
110 | id INTEGER PRIMARY KEY,
111 | application_id INTEGER,
112 | name TEXT,
113 | CONSTRAINT fk_applications
114 | FOREIGN KEY (application_id)
115 | REFERENCES applications(id)
116 | ON DELETE CASCADE
117 | );
118 |
119 | CREATE TABLE IF NOT EXISTS columns(
120 | id INTEGER PRIMARY KEY,
121 | table_id INTEGER,
122 | name TEXT,
123 | type TEXT,
124 | CONSTRAINT fk_tables
125 | FOREIGN KEY (table_id)
126 | REFERENCES tables(id)
127 | ON DELETE CASCADE
128 | );
129 |
130 | DROP VIEW IF EXISTS query_log;
131 | CREATE VIEW query_log AS
132 | SELECT q.id AS id,
133 | q.timestamp AS timestamp,
134 | (CASE WHEN q.sql = q.original_sql THEN 'NO'
135 | WHEN q.sql != q.original_sql THEN 'YES'
136 | END) AS rewritten,
137 | (SELECT AVG(q1.query_time_ms) FROM queries q1 WHERE q1.sql=q.original_sql) AS before_latency,
138 | q.query_time_ms AS after_latency,
139 | q.original_sql AS sql,
140 | (CASE WHEN s.query_id IS NOT NULL THEN 'YES' ELSE 'NO'
141 | END) AS suggestion,
142 | (CASE WHEN s.query_id IS NOT NULL THEN s.query_time_ms ELSE -1000
143 | END) AS suggested_latency,
144 | a.user_id AS user_id,
145 | a.name AS app_name
146 | FROM queries q
147 | JOIN applications a ON q.appguid = a.guid
148 | LEFT OUTER JOIN suggestions s ON q.id = s.query_id
149 | ORDER BY q.timestamp DESC;
--------------------------------------------------------------------------------
/server/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ISG-ICS/QueryBooster/44365a3786e66d6b98083952cc68491060a277ae/server/__init__.py
--------------------------------------------------------------------------------
/server/wsgi.py:
--------------------------------------------------------------------------------
1 | from server import app
2 |
3 | if __name__ == "__main__":
4 | app.run()
--------------------------------------------------------------------------------
/sql/example1.sql:
--------------------------------------------------------------------------------
1 | SELECT S.name, B.make, B.model
2 | FROM Sailors S, Reserves R, Boats B
3 | WHERE S.id = R.sid
4 | AND R.bid = B.id
5 | AND R.date = '2001-12-25'
6 | AND B.color = 'red';
7 |
--------------------------------------------------------------------------------
/sql/rule0_left.sql:
--------------------------------------------------------------------------------
1 | SELECT A, MAX(DISTINCT (SELECT B FROM R WHERE C = 0)), D
2 | FROM S;
3 |
--------------------------------------------------------------------------------
/sql/rule0_left.svg:
--------------------------------------------------------------------------------
1 |
2 | value A select[0] value B select C eq[0] 0 eq[1] where from R distinct max value select[1] value D select[2] from S value A select[0] value B select C eq[0] 0 eq[1] where from R max value select[1] value D select[2] from S value e1.name select[0] value e1.age select[1] value e2.salary select[2] value employee name e1 from[0] value employee name e2 from[1] e1.id eq[0] e2.id eq[1] and[0] e1.age gt[0] 17 gt[1] and[1] e2.salary gt[0] 35000 gt[1] and[2] where value e1.name select[0] value e1.age select[1] value e1.salary select[2] value employee name e1 from e1.age gt[0] 17 gt[1] and[0] e1.salary gt[0] 35000 gt[1] and[1] where value empno select[0] value firstnme select[1] value lastname select[2] value phoneno select[3] workdept in[0] value deptno select deptname eq[0] literal OPERATIONS eq[1] where from department in[1] where from employee value empno select_distinct[0] value firstnme select_distinct[1] value lastname select_distinct[2] value phoneno select_distinct[3] value employee name emp from[0] value department name dept from[1] emp.workdept eq[0] dept.deptno eq[1] and[0] dept.deptname eq[0] literal OPERATIONS eq[1] and[1] where value UserId select_distinct[0] FirstBadgeDate eq[0] min Date value select value Badges name i from o.UserId eq[0] i.UserId eq[1] where eq[1] value select_distinct[1] value Badges name o from value o.UserId select_distinct[0] value FirstBadgeDate select_distinct[1] value Badges name o from[0] value UserId select[0] min Date value name FirstBadgeDate select[1] value UserId groupby from Badges value name i inner join o.UserId eq[0] i.UserId eq[1] on from[1] value o.UserId select_distinct[0] value FirstBadgeDate select_distinct[1] value Badges name o from[0] value UserId select[0] min Date value name FirstBadgeDate select[1] value UserId groupby from Badges value name i inner join o.UserId eq[0] i.UserId eq[1] on from[1] value UserId select[0] min Date value name FirstBadgeDate select[1] value UserId groupby from Badges tweets.created_at cast[0] date cast[1] value select value tweets.created_at select = e2.
16 | AND e1.age > 17
17 | AND e2.salary > 35000
18 | '''
19 | print(StringUtil.strim(input))
20 |
21 | input = '''
22 | SELECT COUNT(.admin_permission_id) AS col_0_0_
23 | FROM
24 | INNER JOIN blc_admin_role_permission_xref AS allroles1_
25 | ON .admin_permission_id = allroles1_.admin_permission_id
26 | INNER JOIN blc_admin_role AS adminrolei2_
27 | ON allroles1_.admin_role_id = adminrolei2_.admin_role_id
28 | WHERE .is_friendly = 1
29 | AND adminrolei2_.admin_role_id = 1
30 | '''
31 | print(StringUtil.strim(input))
32 |
33 | input = '''
34 | SELECT
35 | FROM
36 | WHERE IN (
37 | SELECT
38 | FROM
39 | WHERE =
40 | )
41 | '''
42 | print(StringUtil.strim(input))
--------------------------------------------------------------------------------
/tests/test_query_patcher.py:
--------------------------------------------------------------------------------
1 | from core.query_patcher import QueryPatcher
2 |
3 |
4 | def test_patch_ilike():
5 | q0 = "SELECT * FROM tweets WHERE ILIKE(text, '%iphone%')"
6 | q1 = "SELECT * FROM tweets WHERE text ILIKE '%iphone%'"
7 | assert q1 == QueryPatcher.patch_ilike(q0)
8 |
9 |
10 | def test_patch_timestamp_1():
11 | q0 = '''
12 | SELECT tweets.state_name AS state_name
13 | FROM public.tweets AS tweets
14 | WHERE DATE_TRUNC('QUARTER', tweets.created_at) = TIMESTAMP('2017-04-01 00:00:00.000')
15 | AND tweets.text ILIKE '%iphone%'
16 | GROUP BY 1
17 | '''
18 | q1 = '''
19 | SELECT tweets.state_name AS state_name
20 | FROM public.tweets AS tweets
21 | WHERE DATE_TRUNC('QUARTER', tweets.created_at) = (TIMESTAMP '2017-04-01 00:00:00.000')
22 | AND tweets.text ILIKE '%iphone%'
23 | GROUP BY 1
24 | '''
25 | assert q1 == QueryPatcher.patch_timestamp(q0)
26 |
27 |
28 | def test_patch_timestamp_2():
29 | q0 = "SELECT SUM(1), CAST(state_name AS TEXT) FROM tweets WHERE DATE_TRUNC('QUARTER', created_at) IN (TIMESTAMP('2016-10-01 00:00:00.000'), TIMESTAMP('2017-01-01 00:00:00.000'), TIMESTAMP('2017-04-01 00:00:00.000')) AND STRPOS(text, 'iphone') > 0 GROUP BY 2"
30 | q1 = "SELECT SUM(1), CAST(state_name AS TEXT) FROM tweets WHERE DATE_TRUNC('QUARTER', created_at) IN ((TIMESTAMP '2016-10-01 00:00:00.000'), (TIMESTAMP '2017-01-01 00:00:00.000'), (TIMESTAMP '2017-04-01 00:00:00.000')) AND STRPOS(text, 'iphone') > 0 GROUP BY 2"
31 | assert q1 == QueryPatcher.patch_timestamp(q0)
--------------------------------------------------------------------------------
/tests/test_rule_parser.py:
--------------------------------------------------------------------------------
1 | from core.rule_parser import RuleParser
2 | from core.rule_parser import Scope
3 |
4 | def test_extendToFullSQL():
5 |
6 | # CONDITION scope
7 | pattern = 'CAST(V1 AS DATE)'
8 | rewrite = 'V1'
9 | pattern, scope = RuleParser.extendToFullSQL(pattern)
10 | assert pattern == 'SELECT * FROM t WHERE CAST(V1 AS DATE)'
11 | assert scope == Scope.CONDITION
12 | rewrite, scope = RuleParser.extendToFullSQL(rewrite)
13 | assert rewrite == 'SELECT * FROM t WHERE V1'
14 | assert scope == Scope.CONDITION
15 |
16 | # WHERE scope
17 | pattern = 'WHERE CAST(V1 AS DATE)'
18 | rewrite = 'WHERE V1'
19 | pattern, scope = RuleParser.extendToFullSQL(pattern)
20 | assert pattern == 'SELECT * FROM t WHERE CAST(V1 AS DATE)'
21 | assert scope == Scope.WHERE
22 | rewrite, scope = RuleParser.extendToFullSQL(rewrite)
23 | assert rewrite == 'SELECT * FROM t WHERE V1'
24 | assert scope == Scope.WHERE
25 |
26 | # FROM scope
27 | pattern = 'FROM lineitem'
28 | rewrite = 'FROM v_lineitem'
29 | pattern, scope = RuleParser.extendToFullSQL(pattern)
30 | assert pattern == 'SELECT * FROM lineitem'
31 | assert scope == Scope.FROM
32 | rewrite, scope = RuleParser.extendToFullSQL(rewrite)
33 | assert rewrite == 'SELECT * FROM v_lineitem'
34 | assert scope == Scope.FROM
35 |
36 | # SELECT scope with FROM and WHERE
37 | pattern = '''
38 | select VL1
39 | from V1 V2,
40 | V3 V4
41 | where V2.V6=V4.V8
42 | and VL2
43 | '''
44 | rewrite = '''
45 | select VL1
46 | from V1 V2
47 | where VL2
48 | '''
49 | pattern, scope = RuleParser.extendToFullSQL(pattern)
50 | assert pattern == '''
51 | select VL1
52 | from V1 V2,
53 | V3 V4
54 | where V2.V6=V4.V8
55 | and VL2
56 | '''
57 | assert scope == Scope.SELECT
58 | rewrite, scope = RuleParser.extendToFullSQL(rewrite)
59 | assert rewrite == '''
60 | select VL1
61 | from V1 V2
62 | where VL2
63 | '''
64 | assert scope == Scope.SELECT
65 |
66 | # SELECT scope with FROM
67 | pattern = 'SELECT VL1 FROM lineitem'
68 | rewrite = 'SELECT VL1 FROM v_lineitem'
69 | pattern, scope = RuleParser.extendToFullSQL(pattern)
70 | assert pattern == 'SELECT VL1 FROM lineitem'
71 | assert scope == Scope.SELECT
72 | rewrite, scope = RuleParser.extendToFullSQL(rewrite)
73 | assert rewrite == 'SELECT VL1 FROM v_lineitem'
74 | assert scope == Scope.SELECT
75 |
76 | # SELECT scope with only SELECT
77 | pattern = 'SELECT CAST(V1 AS DATE)'
78 | rewrite = 'SELECT V1'
79 | pattern, scope = RuleParser.extendToFullSQL(pattern)
80 | assert pattern == 'SELECT CAST(V1 AS DATE)'
81 | assert scope == Scope.SELECT
82 | rewrite, scope = RuleParser.extendToFullSQL(rewrite)
83 | assert rewrite == 'SELECT V1'
84 | assert scope == Scope.SELECT
85 |
86 | def test_replaceVars():
87 |
88 | # single var case
89 | pattern = 'CAST( AS DATE)'
90 | rewrite = ''
91 | pattern, rewrite, mapping = RuleParser.replaceVars(pattern, rewrite)
92 | assert pattern == 'CAST(V001 AS DATE)'
93 | assert rewrite == 'V001'
94 |
95 | # multiple var and varList case
96 | pattern = '''
97 | select <>
98 | from ,
99 |
100 | where .=.
101 | and <>
102 | '''
103 | rewrite = '''
104 | select <>
105 | from
106 | where <>
107 | '''
108 | pattern, rewrite, mapping = RuleParser.replaceVars(pattern, rewrite)
109 | assert pattern == '''
110 | select VL001
111 | from V001 V002,
112 | V003 V004
113 | where V002.V005=V004.V006
114 | and VL002
115 | '''
116 | assert rewrite == '''
117 | select VL001
118 | from V001 V002
119 | where VL002
120 | '''
121 |
122 | def test_parse():
123 |
124 | # Init test_rules
125 | test_rules = []
126 | # Rule 1:
127 | rule = {
128 | 'pattern': 'CAST( AS DATE)',
129 | 'rewrite': ''
130 | }
131 | internal_rule = {
132 | 'pattern_json': '{"cast": ["V001", {"date": {}}]}',
133 | 'rewrite_json': '"V001"'
134 | }
135 | test_rules.append((rule, internal_rule))
136 | # Rule 2:
137 | rule = {
138 | 'pattern': "STRPOS(LOWER(), '') > 0",
139 | 'rewrite': " ILIKE '%%'"
140 | }
141 | internal_rule = {
142 | 'pattern_json': '{"gt": [{"strpos": [{"lower": "V001"}, {"literal": "V002"}]}, 0]}',
143 | 'rewrite_json': '{"ilike": ["V001", {"literal": "%V002%"}]}'
144 | }
145 | test_rules.append((rule, internal_rule))
146 |
147 | # Test test_rules
148 | for rule, internal_rule in test_rules:
149 | pattern_json, rewrite_json, mapping = RuleParser.parse(rule['pattern'], rule['rewrite'])
150 | assert pattern_json == internal_rule['pattern_json']
151 | assert rewrite_json == internal_rule['rewrite_json']
152 |
153 |
154 | #incorrect brackets
155 | def test_brackets_1():
156 |
157 | pattern = '''WHERE 11
158 | AND a <= 11
159 | '''
160 |
161 | index = RuleParser.find_malformed_brackets(pattern)
162 | assert index == 6
163 |
164 | #incorrect brackets
165 | def test_brackets_2():
166 |
167 | pattern = '''WHERE 11
168 | AND a <= 11
169 | '''
170 |
171 | index = RuleParser.find_malformed_brackets(pattern)
172 | assert index == 6
173 |
174 | #incorrect brackets
175 | def test_parse_validator_3():
176 |
177 | pattern = '''WHERE 11
178 | AND a <= 11
179 | '''
180 |
181 | index = RuleParser.find_malformed_brackets(pattern)
182 | assert index == 6
183 |
184 | #incorrect brackets
185 | def test_parse_validator_4():
186 |
187 | pattern = '''WHERE [x> > 11
188 | AND a <= 11
189 | '''
190 |
191 | index = RuleParser.find_malformed_brackets(pattern)
192 | assert index == 6
193 |
194 |
195 | #incorrect brackets
196 | def test_parse_validator_5():
197 |
198 | pattern = '''WHERE (x> > 11
199 | AND a <= 11
200 | '''
201 |
202 | index = RuleParser.find_malformed_brackets(pattern)
203 | assert index == 6
204 |
205 | #incorrect brackets
206 | def test_parse_validator_6():
207 |
208 | pattern = '''WHERE {x> > 11
209 | AND a <= 11
210 | '''
211 | index = RuleParser.find_malformed_brackets(pattern)
212 | assert index == 6
213 |
214 |
--------------------------------------------------------------------------------
13 |